diff options
Diffstat (limited to 'gcc')
25 files changed, 841 insertions, 237 deletions
diff --git a/gcc/rust/Make-lang.in b/gcc/rust/Make-lang.in index d625f4f..cb461236 100644 --- a/gcc/rust/Make-lang.in +++ b/gcc/rust/Make-lang.in @@ -89,6 +89,7 @@ GRS_OBJS = \ rust/rust-hir-map.o \ rust/rust-attributes.o \ rust/rust-abi.o \ + rust/rust-macro.o \ rust/rust-ast-lower.o \ rust/rust-ast-lower-base.o \ rust/rust-ast-lower-pattern.o \ diff --git a/gcc/rust/ast/rust-ast-fragment.cc b/gcc/rust/ast/rust-ast-fragment.cc index c491609..fba629c 100644 --- a/gcc/rust/ast/rust-ast-fragment.cc +++ b/gcc/rust/ast/rust-ast-fragment.cc @@ -21,8 +21,9 @@ namespace Rust { namespace AST { -Fragment::Fragment (FragmentKind kind, std::vector<SingleASTNode> nodes) - : kind (kind), nodes (std::move (nodes)) +Fragment::Fragment (FragmentKind kind, std::vector<SingleASTNode> nodes, + std::vector<std::unique_ptr<AST::Token>> tokens) + : kind (kind), nodes (std::move (nodes)), tokens (std::move (tokens)) {} Fragment::Fragment (Fragment const &other) : kind (other.get_kind ()) @@ -33,13 +34,17 @@ Fragment::Fragment (Fragment const &other) : kind (other.get_kind ()) Fragment & Fragment::operator= (Fragment const &other) { + kind = other.get_kind (); + nodes.clear (); nodes.reserve (other.nodes.size ()); - kind = other.get_kind (); for (auto &n : other.nodes) - { - nodes.push_back (n); - } + nodes.push_back (n); + + tokens.clear (); + tokens.reserve (other.tokens.size ()); + for (auto &t : other.tokens) + tokens.emplace_back (t->clone_token ()); return *this; } @@ -47,19 +52,20 @@ Fragment::operator= (Fragment const &other) Fragment Fragment::create_error () { - return Fragment (FragmentKind::Error, {}); + return Fragment (FragmentKind::Error, {}, {}); } -Fragment -Fragment::complete (std::vector<AST::SingleASTNode> nodes) -{ - return Fragment (FragmentKind::Complete, std::move (nodes)); -} +Fragment::Fragment (std::vector<AST::SingleASTNode> nodes, + std::vector<std::unique_ptr<AST::Token>> tokens) + : kind (FragmentKind::Complete), nodes (std::move (nodes)), + tokens (std::move (tokens)) +{} -Fragment -Fragment::unexpanded () +Fragment::Fragment (std::vector<AST::SingleASTNode> nodes, + std::unique_ptr<AST::Token> token) + : kind (FragmentKind::Complete), nodes (std::move (nodes)) { - return Fragment (FragmentKind::Unexpanded, {}); + tokens.emplace_back (std::move (token)); } std::vector<SingleASTNode> & @@ -68,6 +74,12 @@ Fragment::get_nodes () return nodes; } +std::vector<std::unique_ptr<AST::Token>> & +Fragment::get_tokens () +{ + return tokens; +} + FragmentKind Fragment::get_kind () const { diff --git a/gcc/rust/ast/rust-ast-fragment.h b/gcc/rust/ast/rust-ast-fragment.h index 3ef4ba1..22e9909 100644 --- a/gcc/rust/ast/rust-ast-fragment.h +++ b/gcc/rust/ast/rust-ast-fragment.h @@ -28,11 +28,6 @@ namespace AST { enum class FragmentKind { /** - * If an AST Fragment still contains unexpanded tokens - this should only be - * used in the case of builtin macros which need to be expanded eagerly. - */ - Unexpanded, - /** * A completely expanded AST Fragment. This signifies that all * `SingleASTNode`s in the `nodes` vector are valid. * @@ -68,15 +63,18 @@ public: /** * Create a complete AST fragment */ - static Fragment complete (std::vector<AST::SingleASTNode> nodes); + Fragment (std::vector<AST::SingleASTNode> nodes, + std::vector<std::unique_ptr<AST::Token>> tokens); /** - * Create a fragment which contains unexpanded nodes + * Create a complete AST fragment made of a single token */ - static Fragment unexpanded (); + Fragment (std::vector<AST::SingleASTNode> nodes, + std::unique_ptr<AST::Token> tok); FragmentKind get_kind () const; std::vector<SingleASTNode> &get_nodes (); + std::vector<std::unique_ptr<AST::Token>> &get_tokens (); bool is_error () const; bool should_expand () const; @@ -90,7 +88,8 @@ public: void accept_vis (ASTVisitor &vis); private: - Fragment (FragmentKind kind, std::vector<SingleASTNode> nodes); + Fragment (FragmentKind kind, std::vector<SingleASTNode> nodes, + std::vector<std::unique_ptr<AST::Token>> tokens); FragmentKind kind; @@ -105,6 +104,12 @@ private: std::vector<SingleASTNode> nodes; /** + * The tokens associated with an AST fragment. This vector represents the + * actual tokens of the various nodes that are part of the fragment. + */ + std::vector<std::unique_ptr<AST::Token>> tokens; + + /** * We need to make a special case for Expression and Type fragments as only * one Node will be extracted from the `nodes` vector */ diff --git a/gcc/rust/ast/rust-ast.cc b/gcc/rust/ast/rust-ast.cc index 813c35b..a5b7f60b 100644 --- a/gcc/rust/ast/rust-ast.cc +++ b/gcc/rust/ast/rust-ast.cc @@ -1323,6 +1323,12 @@ std::string MacroInvocation::as_string () const { std::string str = "MacroInvocation: "; + auto is_builtin = kind == InvocKind::Builtin; + + if (is_builtin) + str += "[builtin] "; + else + str += "[regular] "; str += append_attributes (outer_attrs, OUTER); @@ -1331,6 +1337,16 @@ MacroInvocation::as_string () const str += "\n has semicolon: "; str += has_semicolon () ? "true" : "false"; + if (is_builtin) + { + str += "[PENDING EAGER INVOCATIONS]: "; + for (auto &pending : pending_eager_invocs) + { + str += pending->as_string (); + str += "\n"; + } + } + return str; } diff --git a/gcc/rust/ast/rust-ast.h b/gcc/rust/ast/rust-ast.h index 6ec2841..ffb671d 100644 --- a/gcc/rust/ast/rust-ast.h +++ b/gcc/rust/ast/rust-ast.h @@ -773,6 +773,7 @@ public: DelimTokenTree (DelimTokenTree const &other) : delim_type (other.delim_type), locus (other.locus) { + token_trees.clear (); token_trees.reserve (other.token_trees.size ()); for (const auto &e : other.token_trees) token_trees.push_back (e->clone_token_tree ()); @@ -784,6 +785,7 @@ public: delim_type = other.delim_type; locus = other.locus; + token_trees.clear (); token_trees.reserve (other.token_trees.size ()); for (const auto &e : other.token_trees) token_trees.push_back (e->clone_token_tree ()); @@ -1523,6 +1525,9 @@ public: DelimTokenTree &get_delim_tok_tree () { return token_tree; } const DelimTokenTree &get_delim_tok_tree () const { return token_tree; } + // Set the delim token tree of a macro invocation + void set_delim_tok_tree (DelimTokenTree tree) { token_tree = tree; } + // TODO: this mutable getter seems kinda dodgy SimplePath &get_path () { return path; } const SimplePath &get_path () const { return path; } diff --git a/gcc/rust/ast/rust-item.h b/gcc/rust/ast/rust-item.h index 206887c..4432ed8 100644 --- a/gcc/rust/ast/rust-item.h +++ b/gcc/rust/ast/rust-item.h @@ -22,11 +22,11 @@ #include "rust-ast.h" #include "rust-path.h" #include "rust-common.h" +#include "rust-expr.h" namespace Rust { namespace AST { // forward decls -class BlockExpr; class TypePath; // TODO: inline? diff --git a/gcc/rust/ast/rust-macro.cc b/gcc/rust/ast/rust-macro.cc new file mode 100644 index 0000000..b90cc15 --- /dev/null +++ b/gcc/rust/ast/rust-macro.cc @@ -0,0 +1,64 @@ +// Copyright (C) 2020-2022 Free Software Foundation, Inc. + +// This file is part of GCC. + +// GCC is free software; you can redistribute it and/or modify it under +// the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3, or (at your option) any later +// version. + +// GCC is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. + +// You should have received a copy of the GNU General Public License +// along with GCC; see the file COPYING3. If not see +// <http://www.gnu.org/licenses/>. + +#include "rust-macro.h" + +namespace Rust { +namespace AST { + +BuiltinMacro +builtin_macro_from_string (const std::string &identifier) +{ + if (identifier == "assert") + return BuiltinMacro::Assert; + + if (identifier == "file") + return BuiltinMacro::File; + + if (identifier == "line") + return BuiltinMacro::Line; + + if (identifier == "column") + return BuiltinMacro::Column; + + if (identifier == "include_bytes") + return BuiltinMacro::IncludeBytes; + + if (identifier == "include_str") + return BuiltinMacro::IncludeStr; + + if (identifier == "compile_error") + return BuiltinMacro::CompileError; + + if (identifier == "concat") + return BuiltinMacro::Concat; + + if (identifier == "env") + return BuiltinMacro::Env; + + if (identifier == "cfg") + return BuiltinMacro::Cfg; + + if (identifier == "include") + return BuiltinMacro::Include; + + gcc_unreachable (); +} + +} // namespace AST +} // namespace Rust diff --git a/gcc/rust/ast/rust-macro.h b/gcc/rust/ast/rust-macro.h index 48e4524..e7b0d1f 100644 --- a/gcc/rust/ast/rust-macro.h +++ b/gcc/rust/ast/rust-macro.h @@ -222,7 +222,7 @@ public: }; private: - std::vector<std::unique_ptr<MacroMatch> > matches; + std::vector<std::unique_ptr<MacroMatch>> matches; MacroRepOp op; // bool has_sep; @@ -235,7 +235,7 @@ public: // Returns whether macro match repetition has separator token. bool has_sep () const { return sep != nullptr; } - MacroMatchRepetition (std::vector<std::unique_ptr<MacroMatch> > matches, + MacroMatchRepetition (std::vector<std::unique_ptr<MacroMatch>> matches, MacroRepOp op, std::unique_ptr<MacroRepSep> sep, Location locus) : matches (std::move (matches)), op (op), sep (std::move (sep)), @@ -290,8 +290,8 @@ public: MacroRepOp get_op () const { return op; } const std::unique_ptr<MacroRepSep> &get_sep () const { return sep; } - std::vector<std::unique_ptr<MacroMatch> > &get_matches () { return matches; } - const std::vector<std::unique_ptr<MacroMatch> > &get_matches () const + std::vector<std::unique_ptr<MacroMatch>> &get_matches () { return matches; } + const std::vector<std::unique_ptr<MacroMatch>> &get_matches () const { return matches; } @@ -309,7 +309,7 @@ protected: class MacroMatcher : public MacroMatch { DelimType delim_type; - std::vector<std::unique_ptr<MacroMatch> > matches; + std::vector<std::unique_ptr<MacroMatch>> matches; Location locus; // TODO: think of way to mark invalid that doesn't take up more space @@ -317,7 +317,7 @@ class MacroMatcher : public MacroMatch public: MacroMatcher (DelimType delim_type, - std::vector<std::unique_ptr<MacroMatch> > matches, + std::vector<std::unique_ptr<MacroMatch>> matches, Location locus) : delim_type (delim_type), matches (std::move (matches)), locus (locus), is_invalid (false) @@ -369,8 +369,8 @@ public: } DelimType get_delim_type () const { return delim_type; } - std::vector<std::unique_ptr<MacroMatch> > &get_matches () { return matches; } - const std::vector<std::unique_ptr<MacroMatch> > &get_matches () const + std::vector<std::unique_ptr<MacroMatch>> &get_matches () { return matches; } + const std::vector<std::unique_ptr<MacroMatch>> &get_matches () const { return matches; } @@ -596,6 +596,9 @@ enum class BuiltinMacro Include }; +BuiltinMacro +builtin_macro_from_string (const std::string &identifier); + /* AST node of a macro invocation, which is replaced by the macro result at * compile time. This is technically a sum-type/tagged-union, which represents * both classic macro invocations and builtin macro invocations. Regular macro @@ -634,7 +637,8 @@ public: { return std::unique_ptr<MacroInvocation> ( new MacroInvocation (InvocKind::Regular, Optional<BuiltinMacro>::none (), - invoc_data, outer_attrs, locus, is_semi_coloned)); + invoc_data, outer_attrs, locus, is_semi_coloned, + {})); } /** @@ -642,15 +646,18 @@ public: * name-resolution and within the macro expander, so unless you're modifying * these visitors, you probably do not want to use this function. */ - static std::unique_ptr<MacroInvocation> - Builtin (BuiltinMacro kind, MacroInvocData invoc_data, - std::vector<Attribute> outer_attrs, Location locus, - bool is_semi_coloned = false) + static std::unique_ptr<MacroInvocation> Builtin ( + BuiltinMacro kind, MacroInvocData invoc_data, + std::vector<Attribute> outer_attrs, Location locus, + std::vector<std::unique_ptr<MacroInvocation>> &&pending_eager_invocations + = {}, + bool is_semi_coloned = false) { return std::unique_ptr<MacroInvocation> ( new MacroInvocation (InvocKind::Builtin, Optional<BuiltinMacro>::some (kind), invoc_data, - outer_attrs, locus, is_semi_coloned)); + outer_attrs, locus, is_semi_coloned, + std::move (pending_eager_invocations))); } Location get_locus () const override final { return locus; } @@ -688,18 +695,53 @@ public: InvocKind get_kind () const { return kind; } Optional<BuiltinMacro> get_builtin_kind () const { return builtin_kind; } + /** + * Turn the current MacroInvocation into a builtin macro invocation + */ + void map_to_builtin (BuiltinMacro macro) + { + kind = InvocKind::Builtin; + builtin_kind = Optional<BuiltinMacro>::some (macro); + } + + /** + * Get the list of pending macro invcations within the builtin macro + * invocation that should get expanded eagerly. + */ + std::vector<std::unique_ptr<MacroInvocation>> & + get_pending_eager_invocations () + { + rust_assert (kind == InvocKind::Builtin); + + return pending_eager_invocs; + } + private: /* Full constructor */ - MacroInvocation (InvocKind kind, Optional<BuiltinMacro> builtin_kind, - MacroInvocData invoc_data, - std::vector<Attribute> outer_attrs, Location locus, - bool is_semi_coloned) + MacroInvocation ( + InvocKind kind, Optional<BuiltinMacro> builtin_kind, + MacroInvocData invoc_data, std::vector<Attribute> outer_attrs, + Location locus, bool is_semi_coloned, + std::vector<std::unique_ptr<MacroInvocation>> &&pending_eager_invocs) : outer_attrs (std::move (outer_attrs)), locus (locus), node_id (Analysis::Mappings::get ()->get_next_node_id ()), invoc_data (std::move (invoc_data)), is_semi_coloned (is_semi_coloned), - kind (kind), builtin_kind (builtin_kind) + kind (kind), builtin_kind (builtin_kind), + pending_eager_invocs (std::move (pending_eager_invocs)) {} + MacroInvocation (const MacroInvocation &other) + : outer_attrs (other.outer_attrs), locus (other.locus), + node_id (other.node_id), invoc_data (other.invoc_data), + is_semi_coloned (other.is_semi_coloned), kind (other.kind), + builtin_kind (other.builtin_kind) + { + if (other.kind == InvocKind::Builtin) + for (auto &pending : other.pending_eager_invocs) + pending_eager_invocs.emplace_back ( + pending->clone_macro_invocation_impl ()); + } + std::vector<Attribute> outer_attrs; Location locus; NodeId node_id; @@ -716,6 +758,16 @@ private: /* If it is a builtin macro, which one */ Optional<BuiltinMacro> builtin_kind = Optional<BuiltinMacro>::none (); + /** + * Pending invocations within a builtin macro invocation. This vector is empty + * and should not be accessed for a regular macro invocation. The macro + * invocations within should be name resolved and expanded before the builtin + * macro invocation get expanded again. It is then the role of the expander to + * insert these new tokens properly in the delimited token tree and try the + * builtin transcriber once again. + */ + std::vector<std::unique_ptr<MacroInvocation>> pending_eager_invocs; + protected: /* Use covariance to implement clone function as returning this object rather * than base */ @@ -817,11 +869,10 @@ protected: class MetaItemSeq : public MetaItem { SimplePath path; - std::vector<std::unique_ptr<MetaItemInner> > seq; + std::vector<std::unique_ptr<MetaItemInner>> seq; public: - MetaItemSeq (SimplePath path, - std::vector<std::unique_ptr<MetaItemInner> > seq) + MetaItemSeq (SimplePath path, std::vector<std::unique_ptr<MetaItemInner>> seq) : path (std::move (path)), seq (std::move (seq)) {} @@ -1024,18 +1075,18 @@ struct AttributeParser { private: // TODO: might as well rewrite to use lexer tokens - std::vector<std::unique_ptr<Token> > token_stream; + std::vector<std::unique_ptr<Token>> token_stream; int stream_pos; public: - AttributeParser (std::vector<std::unique_ptr<Token> > token_stream, + AttributeParser (std::vector<std::unique_ptr<Token>> token_stream, int stream_start_pos = 0) : token_stream (std::move (token_stream)), stream_pos (stream_start_pos) {} ~AttributeParser () = default; - std::vector<std::unique_ptr<MetaItemInner> > parse_meta_item_seq (); + std::vector<std::unique_ptr<MetaItemInner>> parse_meta_item_seq (); private: // Parses a MetaItemInner. diff --git a/gcc/rust/expand/rust-attribute-visitor.cc b/gcc/rust/expand/rust-attribute-visitor.cc index 82580740..3e5d793 100644 --- a/gcc/rust/expand/rust-attribute-visitor.cc +++ b/gcc/rust/expand/rust-attribute-visitor.cc @@ -389,8 +389,6 @@ AttrVisitor::visit (AST::ConstGenericParam &) void AttrVisitor::visit (AST::MacroInvocation ¯o_invoc) { - // FIXME: Probably need to check macro_invoc.kind - // initial strip test based on outer attrs expander.expand_cfg_attrs (macro_invoc.get_outer_attrs ()); if (expander.fails_cfg_with_expand (macro_invoc.get_outer_attrs ())) @@ -1124,7 +1122,7 @@ AttrVisitor::visit (AST::CallExpr &expr) stmt->accept_vis (*this); - auto final_fragment = expand_macro_fragment_recursive (); + auto final_fragment = expander.take_expanded_fragment (); if (final_fragment.should_expand ()) { // Remove the current expanded invocation @@ -3423,7 +3421,7 @@ AttrVisitor::visit (AST::BareFunctionType &type) void AttrVisitor::maybe_expand_expr (std::unique_ptr<AST::Expr> &expr) { - auto final_fragment = expand_macro_fragment_recursive (); + auto final_fragment = expander.take_expanded_fragment (); if (final_fragment.should_expand () && final_fragment.is_expression_fragment ()) expr = final_fragment.take_expression_fragment (); @@ -3432,7 +3430,7 @@ AttrVisitor::maybe_expand_expr (std::unique_ptr<AST::Expr> &expr) void AttrVisitor::maybe_expand_type (std::unique_ptr<AST::Type> &type) { - auto final_fragment = expand_macro_fragment_recursive (); + auto final_fragment = expander.take_expanded_fragment (); if (final_fragment.should_expand () && final_fragment.is_type_fragment ()) type = final_fragment.take_type_fragment (); } diff --git a/gcc/rust/expand/rust-attribute-visitor.h b/gcc/rust/expand/rust-attribute-visitor.h index c2e25df..f153046 100644 --- a/gcc/rust/expand/rust-attribute-visitor.h +++ b/gcc/rust/expand/rust-attribute-visitor.h @@ -44,39 +44,6 @@ public: void expand_trait_method_decl (AST::TraitMethodDecl &decl); /** - * Expand The current macro fragment recursively until it could not be - * expanded further. - * - * The return value checking works because correctly - * expanded fragment can never be an error (if the fragment can not be - * expanded, a stand-in error fragment will be returned; for fragments that - * could not be further expanded: the fragment prior to the expansion failure - * will be returned). - * - * @return Either the expanded fragment or an empty errored-out fragment - * indicating an expansion failure. - */ - AST::Fragment expand_macro_fragment_recursive () - { - auto fragment = expander.take_expanded_fragment (*this); - unsigned int original_depth = expander.expansion_depth; - auto final_fragment = AST::Fragment::create_error (); - - while (fragment.should_expand ()) - { - final_fragment = std::move (fragment); - expander.expansion_depth++; - // further expand the previously expanded macro fragment - auto new_fragment = expander.take_expanded_fragment (*this); - if (new_fragment.is_error ()) - break; - fragment = std::move (new_fragment); - } - expander.expansion_depth = original_depth; - return final_fragment; - } - - /** * Expand a set of values, erasing them if they are marked for strip, and * replacing them with expanded macro nodes if necessary. * This function is slightly different from `expand_pointer_allow_strip` as @@ -101,8 +68,7 @@ public: // mark for stripping if required value->accept_vis (*this); - // recursively expand the children - auto final_fragment = expand_macro_fragment_recursive (); + auto final_fragment = expander.take_expanded_fragment (); if (final_fragment.should_expand ()) { diff --git a/gcc/rust/expand/rust-macro-builtins.cc b/gcc/rust/expand/rust-macro-builtins.cc index f6ddb1a..1caba51 100644 --- a/gcc/rust/expand/rust-macro-builtins.cc +++ b/gcc/rust/expand/rust-macro-builtins.cc @@ -29,6 +29,16 @@ namespace Rust { namespace { + +/** + * Shorthand function for creating unique_ptr tokens + */ +static std::unique_ptr<AST::Token> +make_token (const TokenPtr tok) +{ + return std::unique_ptr<AST::Token> (new AST::Token (tok)); +} + std::unique_ptr<AST::Expr> make_string (Location locus, std::string value) { @@ -37,8 +47,64 @@ make_string (Location locus, std::string value) PrimitiveCoreType::CORETYPE_STR, {}, locus)); } -/* Match the end token of a macro given the start delimiter of the macro */ +// TODO: Is this correct? +static AST::Fragment +make_eager_builtin_invocation ( + AST::BuiltinMacro kind, Location locus, AST::DelimTokenTree arguments, + std::vector<std::unique_ptr<AST::MacroInvocation>> &&pending_invocations) +{ + std::string path_str; + switch (kind) + { + // TODO: Should this be a table lookup? + case AST::BuiltinMacro::Assert: + path_str = "assert"; + break; + case AST::BuiltinMacro::File: + path_str = "file"; + break; + case AST::BuiltinMacro::Line: + path_str = "line"; + break; + case AST::BuiltinMacro::Column: + path_str = "column"; + break; + case AST::BuiltinMacro::IncludeBytes: + path_str = "include_bytes"; + break; + case AST::BuiltinMacro::IncludeStr: + path_str = "include_str"; + break; + case AST::BuiltinMacro::CompileError: + path_str = "compile_error"; + break; + case AST::BuiltinMacro::Concat: + path_str = "concat"; + break; + case AST::BuiltinMacro::Env: + path_str = "env"; + break; + case AST::BuiltinMacro::Cfg: + path_str = "cfg"; + break; + case AST::BuiltinMacro::Include: + path_str = "include"; + break; + } + + std::unique_ptr<AST::Expr> node = AST::MacroInvocation::Builtin ( + kind, + AST::MacroInvocData (AST::SimplePath ( + {AST::SimplePathSegment (path_str, locus)}), + std::move (arguments)), + {}, locus, std::move (pending_invocations)); + + return AST::Fragment ({AST::SingleASTNode (std::move (node))}, + arguments.to_token_stream ()); +} + +/* Match the end token of a macro given the start delimiter of the macro */ static inline TokenId macro_end_token (AST::DelimTokenTree &invoc_token_tree, Parser<MacroInvocLexer> &parser) @@ -64,22 +130,7 @@ macro_end_token (AST::DelimTokenTree &invoc_token_tree, return last_token_id; } -/* Expand and extract an expression from the macro */ - -static inline AST::Fragment -try_expand_macro_expression (AST::Expr *expr, MacroExpander *expander) -{ - rust_assert (expander); - - auto attr_visitor = Rust::AttrVisitor (*expander); - auto early_name_resolver = Resolver::EarlyNameResolver (); - expr->accept_vis (early_name_resolver); - expr->accept_vis (attr_visitor); - return expander->take_expanded_fragment (attr_visitor); -} - /* Expand and then extract a string literal from the macro */ - static std::unique_ptr<AST::LiteralExpr> try_extract_string_literal_from_fragment (const Location &parent_locus, std::unique_ptr<AST::Expr> &node) @@ -97,22 +148,6 @@ try_extract_string_literal_from_fragment (const Location &parent_locus, static_cast<AST::LiteralExpr *> (node->clone_expr ().release ())); } -static std::unique_ptr<AST::LiteralExpr> -try_expand_single_string_literal (AST::Expr *input_expr, - MacroExpander *expander) -{ - auto nodes = try_expand_macro_expression (input_expr, expander); - if (nodes.is_error () || nodes.is_expression_fragment ()) - { - rust_error_at (input_expr->get_locus (), - "argument must be a string literal"); - return nullptr; - } - auto expr = nodes.take_expression_fragment (); - return try_extract_string_literal_from_fragment (input_expr->get_locus (), - expr); -} - static std::vector<std::unique_ptr<AST::Expr>> try_expand_many_expr (Parser<MacroInvocLexer> &parser, const TokenId last_token_id, MacroExpander *expander, @@ -140,22 +175,7 @@ try_expand_many_expr (Parser<MacroInvocLexer> &parser, auto expr = parser.parse_expr (AST::AttrVec (), restrictions); // something must be so wrong that the expression could not be parsed rust_assert (expr); - auto nodes = try_expand_macro_expression (expr.get (), expander); - if (nodes.is_error ()) - { - // not macro - result.push_back (std::move (expr)); - } - else if (!nodes.is_expression_fragment ()) - { - rust_error_at (expr->get_locus (), "expected expression"); - has_error = true; - return empty_expr; - } - else - { - result.push_back (nodes.take_expression_fragment ()); - } + result.push_back (std::move (expr)); auto next_token = parser.peek_current_token (); if (!parser.skip_token (COMMA) && next_token->get_id () != last_token_id) @@ -199,12 +219,7 @@ parse_single_string_literal (AST::DelimTokenTree &invoc_token_tree, else if (parser.peek_current_token ()->get_id () == last_token_id) rust_error_at (invoc_locus, "macro takes 1 argument"); else - { - // when the expression does not seem to be a string literal, we then try - // to parse/expand it as macro to see if it expands to a string literal - auto expr = parser.parse_expr (); - lit_expr = try_expand_single_string_literal (expr.get (), expander); - } + rust_error_at (invoc_locus, "argument must be a string literal"); parser.skip_token (last_token_id); @@ -276,8 +291,10 @@ MacroBuiltin::file_handler (Location invoc_locus, AST::MacroInvocData &) auto current_file = Session::get_instance ().linemap->location_file (invoc_locus); auto file_str = AST::SingleASTNode (make_string (invoc_locus, current_file)); + auto str_token + = make_token (Token::make_string (invoc_locus, std::move (current_file))); - return AST::Fragment::complete ({file_str}); + return AST::Fragment ({file_str}, std::move (str_token)); } AST::Fragment @@ -286,11 +303,13 @@ MacroBuiltin::column_handler (Location invoc_locus, AST::MacroInvocData &) auto current_column = Session::get_instance ().linemap->location_to_column (invoc_locus); + auto column_tok = make_token ( + Token::make_int (invoc_locus, std::to_string (current_column))); auto column_no = AST::SingleASTNode (std::unique_ptr<AST::Expr> ( new AST::LiteralExpr (std::to_string (current_column), AST::Literal::INT, PrimitiveCoreType::CORETYPE_U32, {}, invoc_locus))); - return AST::Fragment::complete ({column_no}); + return AST::Fragment ({column_no}, std::move (column_tok)); } /* Expand builtin macro include_bytes!("filename"), which includes the contents @@ -316,14 +335,25 @@ MacroBuiltin::include_bytes_handler (Location invoc_locus, /* Is there a more efficient way to do this? */ std::vector<std::unique_ptr<AST::Expr>> elts; + + // We create the tokens for a borrow expression of a byte array, so + // & [ <byte0>, <byte1>, ... ] + std::vector<std::unique_ptr<AST::Token>> toks; + toks.emplace_back (make_token (Token::make (AMP, invoc_locus))); + toks.emplace_back (make_token (Token::make (LEFT_SQUARE, invoc_locus))); + for (uint8_t b : bytes) { elts.emplace_back ( new AST::LiteralExpr (std::string (1, (char) b), AST::Literal::BYTE, PrimitiveCoreType::CORETYPE_U8, {} /* outer_attrs */, invoc_locus)); + toks.emplace_back (make_token (Token::make_byte_char (invoc_locus, b))); + toks.emplace_back (make_token (Token::make (COMMA, invoc_locus))); } + toks.emplace_back (make_token (Token::make (RIGHT_SQUARE, invoc_locus))); + auto elems = std::unique_ptr<AST::ArrayElems> ( new AST::ArrayElemsValues (std::move (elts), invoc_locus)); @@ -334,8 +364,9 @@ MacroBuiltin::include_bytes_handler (Location invoc_locus, new AST::BorrowExpr (std::move (array), false, false, {}, invoc_locus)); auto node = AST::SingleASTNode (std::move (borrow)); - return AST::Fragment::complete ({node}); -} + + return AST::Fragment ({node}, std::move (toks)); +} // namespace Rust /* Expand builtin macro include_str!("filename"), which includes the contents of the given file as a string. The file must be UTF-8 encoded. Yields an @@ -362,7 +393,10 @@ MacroBuiltin::include_str_handler (Location invoc_locus, std::string str ((const char *) &bytes[0], bytes.size ()); auto node = AST::SingleASTNode (make_string (invoc_locus, str)); - return AST::Fragment::complete ({node}); + auto str_tok = make_token (Token::make_string (invoc_locus, std::move (str))); + + // FIXME: Do not return an empty token vector here + return AST::Fragment ({node}, std::move (str_tok)); } /* Expand builtin macro compile_error!("error"), which forces a compile error @@ -383,9 +417,62 @@ MacroBuiltin::compile_error_handler (Location invoc_locus, return AST::Fragment::create_error (); } +static std::vector<std::unique_ptr<AST::MacroInvocation>> +check_for_eager_invocations ( + std::vector<std::unique_ptr<AST::Expr>> &expressions) +{ + std::vector<std::unique_ptr<AST::MacroInvocation>> pending; + + for (auto &expr : expressions) + if (expr->get_ast_kind () == AST::Kind::MACRO_INVOCATION) + pending.emplace_back (std::unique_ptr<AST::MacroInvocation> ( + static_cast<AST::MacroInvocation *> (expr->clone_expr ().release ()))); + + return pending; +} + /* Expand builtin macro concat!(), which joins all the literal parameters into a string with no delimiter. */ +// This is a weird one. We want to do something where, if something cannot be +// expanded yet (i.e. macro invocation?) we return the whole MacroInvocation +// node again but expanded as much as possible. +// Is that possible? How do we do that? +// +// Let's take a few examples: +// +// 1. concat!(1, 2, true); +// 2. concat!(a!(), 2, true); +// 3. concat!(concat!(1, false), 2, true); +// 4. concat!(concat!(1, a!()), 2, true); +// +// 1. We simply want to return the new fragment: "12true" +// 2. We want to return `concat!(a_expanded, 2, true)` as a fragment +// 3. We want to return `concat!(1, false, 2, true)` +// 4. We want to return `concat!(concat!(1, a_expanded), 2, true); +// +// How do we do that? +// +// For each (un)expanded fragment: we check if it is expanded fully +// +// 1. What is expanded fully? +// 2. How to check? +// +// If it is expanded fully and not a literal, then we error out. +// Otherwise we simply emplace it back and keep going. +// +// In the second case, we must mark that this concat invocation still has some +// expansion to do: This allows us to return a `MacroInvocation { ... }` as an +// AST fragment, instead of a completed string. +// +// This means that we must change all the `try_expand_many_*` APIs and so on to +// return some sort of index or way to signify that we might want to reuse some +// bits and pieces of the original token tree. +// +// Now, before that: How do we resolve the names used in a builtin macro +// invocation? +// Do we split the two passes of parsing the token tree and then expanding it? +// Can we do that easily? AST::Fragment MacroBuiltin::concat_handler (Location invoc_locus, AST::MacroInvocData &invoc) { @@ -398,12 +485,25 @@ MacroBuiltin::concat_handler (Location invoc_locus, AST::MacroInvocData &invoc) auto last_token_id = macro_end_token (invoc_token_tree, parser); + auto start = lex.get_offs (); /* NOTE: concat! could accept no argument, so we don't have any checks here */ auto expanded_expr = try_expand_many_expr (parser, last_token_id, invoc.get_expander (), has_error); + auto end = lex.get_offs (); + + auto tokens = lex.get_token_slice (start, end); + + auto pending_invocations = check_for_eager_invocations (expanded_expr); + if (!pending_invocations.empty ()) + return make_eager_builtin_invocation (AST::BuiltinMacro::Concat, + invoc_locus, + invoc.get_delim_tok_tree (), + std::move (pending_invocations)); + for (auto &expr : expanded_expr) { - if (!expr->is_literal ()) + if (!expr->is_literal () + && expr->get_ast_kind () != AST::MACRO_INVOCATION) { has_error = true; rust_error_at (expr->get_locus (), "expected a literal"); @@ -431,12 +531,13 @@ MacroBuiltin::concat_handler (Location invoc_locus, AST::MacroInvocData &invoc) return AST::Fragment::create_error (); auto node = AST::SingleASTNode (make_string (invoc_locus, str)); - return AST::Fragment::complete ({node}); + auto str_tok = make_token (Token::make_string (invoc_locus, std::move (str))); + + return AST::Fragment ({node}, std::move (str_tok)); } /* Expand builtin macro env!(), which inspects an environment variable at compile time. */ - AST::Fragment MacroBuiltin::env_handler (Location invoc_locus, AST::MacroInvocData &invoc) { @@ -449,10 +550,22 @@ MacroBuiltin::env_handler (Location invoc_locus, AST::MacroInvocData &invoc) std::unique_ptr<AST::LiteralExpr> lit_expr = nullptr; bool has_error = false; + auto start = lex.get_offs (); auto expanded_expr = try_expand_many_expr (parser, last_token_id, invoc.get_expander (), has_error); + auto end = lex.get_offs (); + + auto tokens = lex.get_token_slice (start, end); + if (has_error) return AST::Fragment::create_error (); + + auto pending = check_for_eager_invocations (expanded_expr); + if (!pending.empty ()) + return make_eager_builtin_invocation (AST::BuiltinMacro::Env, invoc_locus, + invoc_token_tree, + std::move (pending)); + if (expanded_expr.size () < 1 || expanded_expr.size () > 2) { rust_error_at (invoc_locus, "env! takes 1 or 2 arguments"); @@ -492,7 +605,11 @@ MacroBuiltin::env_handler (Location invoc_locus, AST::MacroInvocData &invoc) } auto node = AST::SingleASTNode (make_string (invoc_locus, env_value)); - return AST::Fragment::complete ({node}); + auto tok + = make_token (Token::make_string (invoc_locus, std::move (env_value))); + + // FIXME: Do not return an empty token vector here + return AST::Fragment ({node}, std::move (tok)); } AST::Fragment @@ -527,8 +644,11 @@ MacroBuiltin::cfg_handler (Location invoc_locus, AST::MacroInvocData &invoc) auto literal_exp = AST::SingleASTNode (std::unique_ptr<AST::Expr> ( new AST::LiteralExpr (result ? "true" : "false", AST::Literal::BOOL, PrimitiveCoreType::CORETYPE_BOOL, {}, invoc_locus))); + auto tok = make_token ( + Token::make (result ? TRUE_LITERAL : FALSE_LITERAL, invoc_locus)); - return AST::Fragment::complete ({literal_exp}); + // FIXME: Do not return an empty token vector here + return AST::Fragment ({literal_exp}, std::move (tok)); } /* Expand builtin macro include!(), which includes a source file at the current @@ -585,7 +705,8 @@ MacroBuiltin::include_handler (Location invoc_locus, AST::MacroInvocData &invoc) nodes.push_back (node); } - return AST::Fragment::complete (nodes); + // FIXME: Do not return an empty token vector here + return AST::Fragment (nodes, nullptr); } AST::Fragment @@ -597,8 +718,11 @@ MacroBuiltin::line_handler (Location invoc_locus, AST::MacroInvocData &) auto line_no = AST::SingleASTNode (std::unique_ptr<AST::Expr> ( new AST::LiteralExpr (std::to_string (current_line), AST::Literal::INT, PrimitiveCoreType::CORETYPE_U32, {}, invoc_locus))); + auto tok + = make_token (Token::make_int (invoc_locus, std::to_string (current_line))); - return AST::Fragment::complete ({line_no}); + // FIXME: Do not return an empty token vector here + return AST::Fragment ({line_no}, std::move (tok)); } } // namespace Rust diff --git a/gcc/rust/expand/rust-macro-expand.cc b/gcc/rust/expand/rust-macro-expand.cc index 36b3195..597cb6a 100644 --- a/gcc/rust/expand/rust-macro-expand.cc +++ b/gcc/rust/expand/rust-macro-expand.cc @@ -111,6 +111,122 @@ MacroExpander::expand_decl_macro (Location invoc_locus, } void +MacroExpander::expand_eager_invocations (AST::MacroInvocation &invoc) +{ + if (invoc.get_pending_eager_invocations ().empty ()) + return; + + // We have to basically create a new delimited token tree which contains the + // result of one step of expansion. In the case of builtin macros called with + // other macro invocations, such as `concat!("h", 'a', a!())`, we need to + // expand `a!()` before expanding the concat macro. + // This will, ideally, give us a new token tree containing the various + // existing tokens + the result of the expansion of a!(). + // To do this, we "parse" the given token tree to find anything that "looks + // like a macro invocation". Then, we get the corresponding macro invocation + // from the `pending_eager_invocations` vector and expand it. + // Because the `pending_eager_invocations` vector is created in the same order + // that the DelimTokenTree is parsed, we know that the first macro invocation + // within the DelimTokenTree corresponds to the first element in + // `pending_eager_invocations`. The idea is thus to: + // 1. Find a macro invocation in the token tree, noting the index of the start + // token and of the end token + // 2. Get its associated invocation in `pending_eager_invocations` + // 3. Expand that element + // 4. Get the token tree associated with that AST fragment + // 5. Replace the original tokens corresponding to the invocation with the new + // tokens from the fragment + // pseudo-code: + // + // i = 0; + // for tok in dtt: + // if tok is identifier && tok->next() is !: + // start = index(tok); + // l_delim = tok->next()->next(); + // tok = skip_until_r_delim(); + // end = index(tok); + // + // new_tt = expand_eager_invoc(eagers[i++]); + // old_tt[start..end] = new_tt; + + auto dtt = invoc.get_invoc_data ().get_delim_tok_tree (); + auto stream = dtt.to_token_stream (); + std::vector<std::unique_ptr<AST::TokenTree>> new_stream; + size_t current_pending = 0; + + // we need to create a clone of the delimited token tree as the lexer + // expects ownership of the tokens + std::vector<std::unique_ptr<Rust::AST::Token>> dtt_clone; + for (auto &tok : stream) + dtt_clone.emplace_back (tok->clone_token ()); + + MacroInvocLexer lex (std::move (dtt_clone)); + Parser<MacroInvocLexer> parser (lex); + + // we want to build a substitution map - basically, associating a `start` and + // `end` index for each of the pending macro invocations + std::map<std::pair<size_t, size_t>, std::unique_ptr<AST::MacroInvocation> &> + substitution_map; + + for (size_t i = 0; i < stream.size (); i++) + { + // FIXME: Can't these offsets be figure out when we actually parse the + // pending_eager_invocation in the first place? + auto invocation = parser.parse_macro_invocation ({}); + + // if we've managed to parse a macro invocation, we look at the current + // offset and store them in the substitution map. Otherwise, we skip one + // token and try parsing again + if (invocation) + substitution_map.insert ( + {{i, parser.get_token_source ().get_offs ()}, + invoc.get_pending_eager_invocations ()[current_pending++]}); + else + parser.skip_token (stream[i]->get_id ()); + } + + size_t current_idx = 0; + for (auto kv : substitution_map) + { + auto &to_expand = kv.second; + expand_invoc (*to_expand, false); + + auto fragment = take_expanded_fragment (); + auto &new_tokens = fragment.get_tokens (); + + auto start = kv.first.first; + auto end = kv.first.second; + + // We're now going to re-add the tokens to the invocation's token tree. + // 1. Basically, what we want to do is insert all tokens up until the + // beginning of the macro invocation (start). + // 2. Then, we'll insert all of the tokens resulting from the macro + // expansion: These are in `new_tokens`. + // 3. Finally, we'll do that again from + // the end of macro and go back to 1. + + for (size_t i = current_idx; i < start; i++) + new_stream.emplace_back (stream[i]->clone_token ()); + + for (auto &tok : new_tokens) + new_stream.emplace_back (tok->clone_token ()); + + current_idx = end; + } + + // Once all of that is done, we copy the last remaining tokens from the + // original stream + for (size_t i = current_idx; i < stream.size (); i++) + new_stream.emplace_back (stream[i]->clone_token ()); + + auto new_dtt + = AST::DelimTokenTree (dtt.get_delim_type (), std::move (new_stream)); + + invoc.get_pending_eager_invocations ().clear (); + invoc.get_invoc_data ().set_delim_tok_tree (new_dtt); +} + +void MacroExpander::expand_invoc (AST::MacroInvocation &invoc, bool has_semicolon) { if (depth_exceeds_recursion_limit ()) @@ -119,6 +235,9 @@ MacroExpander::expand_invoc (AST::MacroInvocation &invoc, bool has_semicolon) return; } + if (invoc.get_kind () == AST::MacroInvocation::InvocKind::Builtin) + expand_eager_invocations (invoc); + AST::MacroInvocData &invoc_data = invoc.get_invoc_data (); // ?? @@ -151,6 +270,11 @@ MacroExpander::expand_invoc (AST::MacroInvocation &invoc, bool has_semicolon) if (!ok) return; + // We store the last expanded invocation and macro definition for error + // reporting in case the recursion limit is reached + last_invoc = &invoc; + last_def = rules_def; + if (rules_def->is_builtin ()) fragment = rules_def->get_builtin_transcriber () (invoc.get_locus (), invoc_data); @@ -292,7 +416,7 @@ MacroExpander::expand_crate () // mark for stripping if required item->accept_vis (attr_visitor); - auto fragment = take_expanded_fragment (attr_visitor); + auto fragment = take_expanded_fragment (); if (fragment.should_expand ()) { // Remove the current expanded invocation @@ -711,6 +835,9 @@ static AST::Fragment parse_many (Parser<MacroInvocLexer> &parser, TokenId &delimiter, std::function<AST::SingleASTNode ()> parse_fn) { + auto &lexer = parser.get_token_source (); + auto start = lexer.get_offs (); + std::vector<AST::SingleASTNode> nodes; while (true) { @@ -728,8 +855,9 @@ parse_many (Parser<MacroInvocLexer> &parser, TokenId &delimiter, nodes.emplace_back (std::move (node)); } + auto end = lexer.get_offs (); - return AST::Fragment::complete (std::move (nodes)); + return AST::Fragment (std::move (nodes), lexer.get_token_slice (start, end)); } /** @@ -838,11 +966,16 @@ transcribe_many_stmts (Parser<MacroInvocLexer> &parser, TokenId &delimiter) static AST::Fragment transcribe_expression (Parser<MacroInvocLexer> &parser) { + auto &lexer = parser.get_token_source (); + auto start = lexer.get_offs (); + auto expr = parser.parse_expr (); if (expr == nullptr) return AST::Fragment::create_error (); - return AST::Fragment::complete ({std::move (expr)}); + auto end = lexer.get_offs (); + + return AST::Fragment ({std::move (expr)}, lexer.get_token_slice (start, end)); } /** @@ -853,11 +986,16 @@ transcribe_expression (Parser<MacroInvocLexer> &parser) static AST::Fragment transcribe_type (Parser<MacroInvocLexer> &parser) { + auto &lexer = parser.get_token_source (); + auto start = lexer.get_offs (); + auto type = parser.parse_type (true); for (auto err : parser.get_errors ()) err.emit_error (); - return AST::Fragment::complete ({std::move (type)}); + auto end = lexer.get_offs (); + + return AST::Fragment ({std::move (type)}, lexer.get_token_slice (start, end)); } static AST::Fragment diff --git a/gcc/rust/expand/rust-macro-expand.h b/gcc/rust/expand/rust-macro-expand.h index 6253a4e..e440edf 100644 --- a/gcc/rust/expand/rust-macro-expand.h +++ b/gcc/rust/expand/rust-macro-expand.h @@ -231,7 +231,7 @@ struct MacroExpander : cfg (cfg), crate (crate), session (session), sub_stack (SubstitutionScope ()), expanded_fragment (AST::Fragment::create_error ()), - resolver (Resolver::Resolver::get ()), + has_changed_flag (false), resolver (Resolver::Resolver::get ()), mappings (Analysis::Mappings::get ()) {} @@ -240,6 +240,12 @@ struct MacroExpander // Expands all macros in the crate passed in. void expand_crate (); + /** + * Expand the eager invocations contained within a builtin macro invocation. + * Called by `expand_invoc` when expanding builtin invocations. + */ + void expand_eager_invocations (AST::MacroInvocation &invoc); + /* Expands a macro invocation - possibly make both * have similar duck-typed interface and use templates?*/ // should this be public or private? @@ -315,49 +321,44 @@ struct MacroExpander void set_expanded_fragment (AST::Fragment &&fragment) { + if (!fragment.is_error ()) + has_changed_flag = true; + expanded_fragment = std::move (fragment); } - AST::Fragment take_expanded_fragment (AST::ASTVisitor &vis) + AST::Fragment take_expanded_fragment () { - AST::Fragment old_fragment = std::move (expanded_fragment); - auto accumulator = std::vector<AST::SingleASTNode> (); + auto fragment = std::move (expanded_fragment); expanded_fragment = AST::Fragment::create_error (); - auto early_name_resolver = Resolver::EarlyNameResolver (); - - for (auto &node : old_fragment.get_nodes ()) - { - expansion_depth++; - - node.accept_vis (early_name_resolver); - node.accept_vis (vis); - // we'll decide the next move according to the outcome of the macro - // expansion - if (expanded_fragment.is_error ()) - accumulator.push_back (node); // if expansion fails, there might be a - // non-macro expression we need to keep - else - { - // if expansion succeeded, then we need to merge the fragment with - // the contents in the accumulator, so that our final expansion - // result will contain non-macro nodes as it should - auto new_nodes = expanded_fragment.get_nodes (); - std::move (new_nodes.begin (), new_nodes.end (), - std::back_inserter (accumulator)); - expanded_fragment = AST::Fragment::complete (accumulator); - } - expansion_depth--; - } - - return old_fragment; + + return fragment; } + /** + * Has the MacroExpander expanded a macro since its state was last reset? + */ + bool has_changed () const { return has_changed_flag; } + + /** + * Reset the expander's "changed" state. This function should be executed at + * each iteration in a fixed point loop + */ + void reset_changed_state () { has_changed_flag = false; } + + AST::MacroRulesDefinition *get_last_definition () { return last_def; } + AST::MacroInvocation *get_last_invocation () { return last_invoc; } + private: AST::Crate &crate; Session &session; SubstitutionScope sub_stack; std::vector<ContextType> context; AST::Fragment expanded_fragment; + bool has_changed_flag; + + AST::MacroRulesDefinition *last_def; + AST::MacroInvocation *last_invoc; public: Resolver::Resolver *resolver; diff --git a/gcc/rust/expand/rust-macro-invoc-lexer.cc b/gcc/rust/expand/rust-macro-invoc-lexer.cc index 8a43d29..321f0f9 100644 --- a/gcc/rust/expand/rust-macro-invoc-lexer.cc +++ b/gcc/rust/expand/rust-macro-invoc-lexer.cc @@ -26,4 +26,18 @@ MacroInvocLexer::split_current_token (TokenId new_left __attribute__ ((unused)), // FIXME gcc_unreachable (); } + +std::vector<std::unique_ptr<AST::Token>> +MacroInvocLexer::get_token_slice (size_t start_idx, size_t end_idx) const +{ + std::vector<std::unique_ptr<AST::Token>> slice; + + rust_assert (end_idx < token_stream.size ()); + + for (size_t i = start_idx; i < end_idx; i++) + slice.emplace_back (token_stream[i]->clone_token ()); + + return slice; +} + } // namespace Rust diff --git a/gcc/rust/expand/rust-macro-invoc-lexer.h b/gcc/rust/expand/rust-macro-invoc-lexer.h index 0fd4554..8190d7b 100644 --- a/gcc/rust/expand/rust-macro-invoc-lexer.h +++ b/gcc/rust/expand/rust-macro-invoc-lexer.h @@ -55,6 +55,9 @@ public: size_t get_offs () const { return offs; } + std::vector<std::unique_ptr<AST::Token>> + get_token_slice (size_t start_idx, size_t end_idx) const; + private: size_t offs; std::vector<std::unique_ptr<AST::Token>> token_stream; diff --git a/gcc/rust/parse/rust-parse.h b/gcc/rust/parse/rust-parse.h index 2f767bb..a4b65c5 100644 --- a/gcc/rust/parse/rust-parse.h +++ b/gcc/rust/parse/rust-parse.h @@ -150,6 +150,8 @@ public: std::unique_ptr<AST::TokenTree> parse_token_tree (); AST::Attribute parse_attribute_body (); AST::AttrVec parse_inner_attributes (); + std::unique_ptr<AST::MacroInvocation> + parse_macro_invocation (AST::AttrVec outer_attrs); private: void skip_after_semicolon (); @@ -201,8 +203,6 @@ private: parse_decl_macro_def (AST::Visibility vis, AST::AttrVec outer_attrs); std::unique_ptr<AST::MacroInvocation> parse_macro_invocation_semi (AST::AttrVec outer_attrs); - std::unique_ptr<AST::MacroInvocation> - parse_macro_invocation (AST::AttrVec outer_attrs); AST::MacroRule parse_macro_rule (); AST::MacroMatcher parse_macro_matcher (); std::unique_ptr<AST::MacroMatch> parse_macro_match (); diff --git a/gcc/rust/resolve/rust-early-name-resolver.cc b/gcc/rust/resolve/rust-early-name-resolver.cc index 8100564..60d6c8a 100644 --- a/gcc/rust/resolve/rust-early-name-resolver.cc +++ b/gcc/rust/resolve/rust-early-name-resolver.cc @@ -24,20 +24,17 @@ namespace Rust { namespace Resolver { EarlyNameResolver::EarlyNameResolver () - : resolver (*Resolver::get ()), mappings (*Analysis::Mappings::get ()) + : current_scope (UNKNOWN_NODEID), resolver (*Resolver::get ()), + mappings (*Analysis::Mappings::get ()) {} void EarlyNameResolver::go (AST::Crate &crate) { - // FIXME: Is that valid? Why is the regular name resolution doing - // mappings->get_next_node_id()? - resolver.get_macro_scope ().push (crate.get_node_id ()); - - for (auto &item : crate.items) - item->accept_vis (*this); - - // FIXME: Should we pop the macro scope? + scoped (crate.get_node_id (), [&crate, this] () { + for (auto &item : crate.items) + item->accept_vis (*this); + }); } void @@ -335,11 +332,13 @@ EarlyNameResolver::visit (AST::ClosureExprInner &expr) void EarlyNameResolver::visit (AST::BlockExpr &expr) { - for (auto &stmt : expr.get_statements ()) - stmt->accept_vis (*this); + scoped (expr.get_node_id (), [&expr, this] () { + for (auto &stmt : expr.get_statements ()) + stmt->accept_vis (*this); - if (expr.has_tail_expr ()) - expr.get_tail_expr ()->accept_vis (*this); + if (expr.has_tail_expr ()) + expr.get_tail_expr ()->accept_vis (*this); + }); } void @@ -434,8 +433,11 @@ EarlyNameResolver::visit (AST::WhileLetLoopExpr &expr) void EarlyNameResolver::visit (AST::ForLoopExpr &expr) { - expr.get_iterator_expr ()->accept_vis (*this); - expr.get_loop_block ()->accept_vis (*this); + scoped (expr.get_node_id (), [&expr, this] () { + expr.get_pattern ()->accept_vis (*this); + expr.get_iterator_expr ()->accept_vis (*this); + expr.get_loop_block ()->accept_vis (*this); + }); } void @@ -473,7 +475,9 @@ void EarlyNameResolver::visit (AST::IfLetExpr &expr) { expr.get_value_expr ()->accept_vis (*this); - expr.get_if_block ()->accept_vis (*this); + + scoped (expr.get_node_id (), + [&expr, this] () { expr.get_if_block ()->accept_vis (*this); }); } void @@ -504,16 +508,21 @@ void EarlyNameResolver::visit (AST::MatchExpr &expr) { expr.get_scrutinee_expr ()->accept_vis (*this); - for (auto &match_arm : expr.get_match_cases ()) - { - if (match_arm.get_arm ().has_match_arm_guard ()) - match_arm.get_arm ().get_guard_expr ()->accept_vis (*this); - for (auto &pattern : match_arm.get_arm ().get_patterns ()) - pattern->accept_vis (*this); + scoped (expr.get_node_id (), [&expr, this] () { + for (auto &arm : expr.get_match_cases ()) + { + scoped (arm.get_node_id (), [&arm, this] () { + if (arm.get_arm ().has_match_arm_guard ()) + arm.get_arm ().get_guard_expr ()->accept_vis (*this); - match_arm.get_expr ()->accept_vis (*this); - } + for (auto &pattern : arm.get_arm ().get_patterns ()) + pattern->accept_vis (*this); + + arm.get_expr ()->accept_vis (*this); + }); + } + }); } void @@ -571,8 +580,10 @@ EarlyNameResolver::visit (AST::Method &method) void EarlyNameResolver::visit (AST::Module &module) { - for (auto &item : module.get_items ()) - item->accept_vis (*this); + scoped (module.get_node_id (), [&module, this] () { + for (auto &item : module.get_items ()) + item->accept_vis (*this); + }); } void @@ -722,8 +733,13 @@ EarlyNameResolver::visit (AST::TraitItemType &) void EarlyNameResolver::visit (AST::Trait &trait) { - for (auto &item : trait.get_trait_items ()) - item->accept_vis (*this); + for (auto &generic : trait.get_generic_params ()) + generic->accept_vis (*this); + + scoped (trait.get_node_id (), [&trait, this] () { + for (auto &item : trait.get_trait_items ()) + item->accept_vis (*this); + }); } void @@ -734,8 +750,10 @@ EarlyNameResolver::visit (AST::InherentImpl &impl) for (auto &generic : impl.get_generic_params ()) generic->accept_vis (*this); - for (auto &item : impl.get_impl_items ()) - item->accept_vis (*this); + scoped (impl.get_node_id (), [&impl, this] () { + for (auto &item : impl.get_impl_items ()) + item->accept_vis (*this); + }); } void @@ -746,8 +764,10 @@ EarlyNameResolver::visit (AST::TraitImpl &impl) for (auto &generic : impl.get_generic_params ()) generic->accept_vis (*this); - for (auto &item : impl.get_impl_items ()) - item->accept_vis (*this); + scoped (impl.get_node_id (), [&impl, this] () { + for (auto &item : impl.get_impl_items ()) + item->accept_vis (*this); + }); } void @@ -772,8 +792,10 @@ EarlyNameResolver::visit (AST::ExternalFunctionItem &item) void EarlyNameResolver::visit (AST::ExternBlock &block) { - for (auto &item : block.get_extern_items ()) - item->accept_vis (*this); + scoped (block.get_node_id (), [&block, this] () { + for (auto &item : block.get_extern_items ()) + item->accept_vis (*this); + }); } void @@ -795,6 +817,15 @@ EarlyNameResolver::visit (AST::MacroRulesDefinition &rules_def) rules_def.get_rule_name ()); resolver.get_macro_scope ().insert (path, rules_def.get_node_id (), rules_def.get_locus ()); + + /* Since the EarlyNameResolver runs multiple time (fixed point algorithm) + * we could be inserting the same macro def over and over again until we + * implement some optimizations */ + // FIXME: ARTHUR: Remove that lookup and add proper optimizations instead + AST::MacroRulesDefinition *tmp = nullptr; + if (mappings.lookup_macro_def (rules_def.get_node_id (), &tmp)) + return; + mappings.insert_macro_def (&rules_def); rust_debug_loc (rules_def.get_locus (), "inserting macro def: [%s]", path.get ().c_str ()); @@ -806,6 +837,10 @@ EarlyNameResolver::visit (AST::MacroInvocation &invoc) auto &invoc_data = invoc.get_invoc_data (); auto has_semicolon = invoc.has_semicolon (); + if (invoc.get_kind () == AST::MacroInvocation::InvocKind::Builtin) + for (auto &pending_invoc : invoc.get_pending_eager_invocations ()) + pending_invoc->accept_vis (*this); + // ?? // switch on type of macro: // - '!' syntax macro (inner switch) @@ -847,6 +882,30 @@ EarlyNameResolver::visit (AST::MacroInvocation &invoc) bool ok = mappings.lookup_macro_def (resolved_node, &rules_def); rust_assert (ok); + auto &outer_attrs = rules_def->get_outer_attrs (); + bool is_builtin + = std::any_of (outer_attrs.begin (), outer_attrs.end (), + [] (AST::Attribute attr) { + return attr.get_path () == "rustc_builtin_macro"; + }); + + if (is_builtin) + { + auto builtin_kind + = AST::builtin_macro_from_string (rules_def->get_rule_name ()); + invoc.map_to_builtin (builtin_kind); + } + + auto attributes = rules_def->get_outer_attrs (); + + /* Since the EarlyNameResolver runs multiple time (fixed point algorithm) + * we could be inserting the same macro def over and over again until we + * implement some optimizations */ + // FIXME: ARTHUR: Remove that lookup and add proper optimizations instead + AST::MacroRulesDefinition *tmp_def = nullptr; + if (mappings.lookup_macro_invocation (invoc, &tmp_def)) + return; + mappings.insert_macro_invocation (invoc, rules_def); } diff --git a/gcc/rust/resolve/rust-early-name-resolver.h b/gcc/rust/resolve/rust-early-name-resolver.h index c53ab9f..c74c452 100644 --- a/gcc/rust/resolve/rust-early-name-resolver.h +++ b/gcc/rust/resolve/rust-early-name-resolver.h @@ -35,6 +35,71 @@ public: void go (AST::Crate &crate); private: + /** + * Execute a lambda within a scope. This is equivalent to calling + * `enter_scope` before your code and `exit_scope` after. This ensures + * no errors can be committed + */ + void scoped (NodeId scope_id, std::function<void ()> fn) + { + auto old_scope = current_scope; + current_scope = scope_id; + resolver.get_macro_scope ().push (scope_id); + resolver.push_new_macro_rib (resolver.get_macro_scope ().peek ()); + + fn (); + + resolver.get_macro_scope ().pop (); + current_scope = old_scope; + } + + /** + * The "scope" we are currently in. + * + * This involves lexical scopes: + * + * ```rust + * // current_scope = crate_id; + * macro_rules! foo { () => {} ) + * + * { + * // current_scope = current_block_id; + * macro_rules! foo { () => { something!(); } } + * } + * // current_scope = crate_id; + * ``` + * + * as well as any sort of scope-like structure that might impact import name + * resolution or macro name resolution: + * + * ```rust + * macro_rules! foo { + * () => { fn empty() {} } + * } + * + * + * trait Foo { + * fn foo() { + * fn inner_foo() { + * macro_rules! foo { () => {} ) + * + * foo!(); + * } + * + * foo!(); + * } + * + * foo!(); + * } + * + * foo!(); + * ``` + */ + NodeId current_scope; + + /* The crate's scope */ + NodeId crate_scope; + Resolver &resolver; Analysis::Mappings &mappings; diff --git a/gcc/rust/rust-session-manager.cc b/gcc/rust/rust-session-manager.cc index 6966ccc..d4e3715 100644 --- a/gcc/rust/rust-session-manager.cc +++ b/gcc/rust/rust-session-manager.cc @@ -827,9 +827,6 @@ Session::injection (AST::Crate &crate) void Session::expansion (AST::Crate &crate) { - /* We need to name resolve macros and imports here */ - Resolver::EarlyNameResolver ().go (crate); - rust_debug ("started expansion"); /* rustc has a modification to windows PATH temporarily here, which may end @@ -839,11 +836,41 @@ Session::expansion (AST::Crate &crate) // if not, would at least have to configure recursion_limit ExpansionCfg cfg; + auto fixed_point_reached = false; + unsigned iterations = 0; + // create extctxt? from parse session, cfg, and resolver? /* expand by calling cxtctxt object's monotonic_expander's expand_crate * method. */ MacroExpander expander (crate, cfg, *this); - expander.expand_crate (); + + while (!fixed_point_reached && iterations < cfg.recursion_limit) + { + /* We need to name resolve macros and imports here */ + Resolver::EarlyNameResolver ().go (crate); + + expander.expand_crate (); + + fixed_point_reached = !expander.has_changed (); + expander.reset_changed_state (); + iterations++; + + if (saw_errors ()) + break; + } + + if (iterations == cfg.recursion_limit) + { + auto last_invoc = expander.get_last_invocation (); + auto last_def = expander.get_last_definition (); + + rust_assert (last_def && last_invoc); + + RichLocation range (last_invoc->get_locus ()); + range.add_range (last_def->get_locus ()); + + rust_error_at (range, "reached recursion limit"); + } // error reporting - check unused macros, get missing fragment specifiers diff --git a/gcc/testsuite/rust/compile/builtin_macro_recurse.rs b/gcc/testsuite/rust/compile/builtin_macro_eager1.rs index 0b516fd..0b516fd 100644 --- a/gcc/testsuite/rust/compile/builtin_macro_recurse.rs +++ b/gcc/testsuite/rust/compile/builtin_macro_eager1.rs diff --git a/gcc/testsuite/rust/compile/builtin_macro_eager2.rs b/gcc/testsuite/rust/compile/builtin_macro_eager2.rs new file mode 100644 index 0000000..21bf95c --- /dev/null +++ b/gcc/testsuite/rust/compile/builtin_macro_eager2.rs @@ -0,0 +1,16 @@ +#[rustc_builtin_macro] +macro_rules! env { + () => {}; +} + +macro_rules! a { + () => { + "__undefined__" + }; +} + +fn main() { + let _ = env!(a!()); // { dg-error "environment variable .__undefined__. not defined" } + let _ = env!(a!(), "custom"); // { dg-error "custom" } + let _ = env!(a!(), a!()); // { dg-error "__undefined__" } +} diff --git a/gcc/testsuite/rust/compile/builtin_macro_recurse2.rs b/gcc/testsuite/rust/compile/builtin_macro_recurse2.rs new file mode 100644 index 0000000..d859920 --- /dev/null +++ b/gcc/testsuite/rust/compile/builtin_macro_recurse2.rs @@ -0,0 +1,20 @@ +// { dg-additional-options "-fdump-tree-gimple" } + +#[rustc_builtin_macro] +macro_rules! concat { + () => {}; +} + +macro_rules! a { + () => { + "hey" + }; + ($($t:tt)*) => { + "ho" + }; +} + +fn main() { + // { dg-final { scan-tree-dump-times {"abheyho"} 1 gimple } } + let _ = concat!("a", 'b', a!(), a!(b c d e f a!()), '\0'); +} diff --git a/gcc/testsuite/rust/compile/macro17.rs b/gcc/testsuite/rust/compile/macro17.rs index 7432165..b50afbe 100644 --- a/gcc/testsuite/rust/compile/macro17.rs +++ b/gcc/testsuite/rust/compile/macro17.rs @@ -1,7 +1,7 @@ macro_rules! rep { - ($a:literal) => { $a }; // { dg-error "reached recursion limit" } - ($a:literal $(, $e:literal)*) => { // { dg-error "reached recursion limit" } - $a + rep!(0 $(, $e)*) // { dg-error "Failed to match" } + ($a:literal) => { $a }; + ($a:literal $(, $e:literal)*) => { + $a + rep!(0 $(, $e)*) // { dg-error "reached recursion limit" } } } diff --git a/gcc/testsuite/rust/compile/macro44.rs b/gcc/testsuite/rust/compile/macro44.rs index 84b2cdb..dabac6f 100644 --- a/gcc/testsuite/rust/compile/macro44.rs +++ b/gcc/testsuite/rust/compile/macro44.rs @@ -11,12 +11,12 @@ mod foo { () => {{}}; } - fn foo_f() { // { dg-warning "function is never used" } + fn foo_f() { foo!(); } - fn bar_f() { // { dg-warning "function is never used" } - baz!(); + fn bar_f() { + baz!(); // { dg-error "unknown macro" } } } diff --git a/gcc/testsuite/rust/compile/macro46.rs b/gcc/testsuite/rust/compile/macro46.rs new file mode 100644 index 0000000..3ef811a --- /dev/null +++ b/gcc/testsuite/rust/compile/macro46.rs @@ -0,0 +1,19 @@ +fn foo() {} + +fn main() { + macro_rules! a { + () => { + foo(); + }; + } + + { + macro_rules! a { + () => { + bar(); + }; + } + } + + a!(); +} |