macros: Add remaining context and improve parsing macro dispatch

This allows us to expand macor invocations in more places, as macro
calls are not limited to statements or expressions. It is quite common
to use macros to abstract writing repetitive boilerplate for type
implementations, for example.

8 files changed, 298 insertions, 91 deletions

diff --git a/gcc/rust/ast/rust-ast.h b/gcc/rust/ast/rust-ast.h
index 3b96534..e33fff2 100644
--- a/gcc/rust/ast/rust-ast.h
+++ b/gcc/rust/ast/rust-ast.h
@@ -1511,18 +1511,45 @@ public:
     EXPRESSION,
     ITEM,
     STMT,
+    EXTERN,
+    TRAIT,
+    IMPL,
   };
 
+private:
+  NodeType kind;
+
+  // FIXME make this a union
+  std::unique_ptr<Expr> expr;
+  std::unique_ptr<Item> item;
+  std::unique_ptr<Stmt> stmt;
+  std::unique_ptr<ExternalItem> external_item;
+  std::unique_ptr<TraitItem> trait_item;
+  std::unique_ptr<InherentImplItem> impl_item;
+
+public:
   SingleASTNode (std::unique_ptr<Expr> expr)
-    : kind (EXPRESSION), expr (std::move (expr)), item (nullptr), stmt (nullptr)
+    : kind (EXPRESSION), expr (std::move (expr))
   {}
 
   SingleASTNode (std::unique_ptr<Item> item)
-    : kind (ITEM), expr (nullptr), item (std::move (item)), stmt (nullptr)
+    : kind (ITEM), item (std::move (item))
   {}
 
   SingleASTNode (std::unique_ptr<Stmt> stmt)
-    : kind (STMT), expr (nullptr), item (nullptr), stmt (std::move (stmt))
+    : kind (STMT), stmt (std::move (stmt))
+  {}
+
+  SingleASTNode (std::unique_ptr<ExternalItem> item)
+    : kind (EXTERN), external_item (std::move (item))
+  {}
+
+  SingleASTNode (std::unique_ptr<TraitItem> item)
+    : kind (TRAIT), trait_item (std::move (item))
+  {}
+
+  SingleASTNode (std::unique_ptr<InherentImplItem> item)
+    : kind (IMPL), impl_item (std::move (item))
   {}
 
   SingleASTNode (SingleASTNode const &other)
@@ -1541,6 +1568,18 @@ public:
       case STMT:
 	stmt = other.stmt->clone_stmt ();
 	break;
+
+      case EXTERN:
+	external_item = other.external_item->clone_external_item ();
+	break;
+
+      case TRAIT:
+	trait_item = other.trait_item->clone_trait_item ();
+	break;
+
+      case IMPL:
+	impl_item = other.impl_item->clone_inherent_impl_item ();
+	break;
       }
   }
 
@@ -1560,6 +1599,18 @@ public:
       case STMT:
 	stmt = other.stmt->clone_stmt ();
 	break;
+
+      case EXTERN:
+	external_item = other.external_item->clone_external_item ();
+	break;
+
+      case TRAIT:
+	trait_item = other.trait_item->clone_trait_item ();
+	break;
+
+      case IMPL:
+	impl_item = other.impl_item->clone_inherent_impl_item ();
+	break;
       }
     return *this;
   }
@@ -1569,7 +1620,7 @@ public:
 
   NodeType get_kind () const { return kind; }
 
-  std::unique_ptr<Expr> &get_inner ()
+  std::unique_ptr<Expr> &get_expr ()
   {
     rust_assert (kind == EXPRESSION);
     return expr;
@@ -1610,6 +1661,24 @@ public:
     return std::move (item);
   }
 
+  std::unique_ptr<TraitItem> take_trait_item ()
+  {
+    rust_assert (!is_error ());
+    return std::move (trait_item);
+  }
+
+  std::unique_ptr<ExternalItem> take_external_item ()
+  {
+    rust_assert (!is_error ());
+    return std::move (external_item);
+  }
+
+  std::unique_ptr<InherentImplItem> take_impl_item ()
+  {
+    rust_assert (!is_error ());
+    return std::move (impl_item);
+  }
+
   void accept_vis (ASTVisitor &vis)
   {
     switch (kind)
@@ -1625,6 +1694,18 @@ public:
       case STMT:
 	stmt->accept_vis (vis);
 	break;
+
+      case EXTERN:
+	external_item->accept_vis (vis);
+	break;
+
+      case TRAIT:
+	trait_item->accept_vis (vis);
+	break;
+
+      case IMPL:
+	impl_item->accept_vis (vis);
+	break;
       }
   }
 
@@ -1638,9 +1719,16 @@ public:
 	return item == nullptr;
       case STMT:
 	return stmt == nullptr;
-      default:
-	return true;
+      case EXTERN:
+	return external_item == nullptr;
+      case TRAIT:
+	return trait_item == nullptr;
+      case IMPL:
+	return impl_item == nullptr;
       }
+
+    gcc_unreachable ();
+    return true;
   }
 
   std::string as_string ()
@@ -1653,18 +1741,17 @@ public:
 	return "Item: " + item->as_string ();
       case STMT:
 	return "Stmt: " + stmt->as_string ();
-      default:
-	return "";
+      case EXTERN:
+	return "External Item: " + external_item->as_string ();
+      case TRAIT:
+	return "Trait Item: " + trait_item->as_string ();
+      case IMPL:
+	return "Impl Item: " + impl_item->as_string ();
       }
-  }
-
-private:
-  NodeType kind;
 
-  // FIXME make this a union
-  std::unique_ptr<Expr> expr;
-  std::unique_ptr<Item> item;
-  std::unique_ptr<Stmt> stmt;
+    gcc_unreachable ();
+    return "";
+  }
 };
 
 /* Basically, a "fragment" that can be incorporated into the AST, created as
diff --git a/gcc/rust/ast/rust-macro.h b/gcc/rust/ast/rust-macro.h
index 1c5d102..5ecd5d7 100644
--- a/gcc/rust/ast/rust-macro.h
+++ b/gcc/rust/ast/rust-macro.h
@@ -460,6 +460,7 @@ class MacroInvocation : public TypeNoBounds,
 			public TraitItem,
 			public TraitImplItem,
 			public InherentImplItem,
+			public ExternalItem,
 			public ExprWithoutBlock
 {
   std::vector<Attribute> outer_attrs;
@@ -537,6 +538,11 @@ protected:
     return clone_macro_invocation_impl ();
   }
 
+  MacroInvocation *clone_external_item_impl () const final override
+  {
+    return clone_macro_invocation_impl ();
+  }
+
   /*virtual*/ MacroInvocation *clone_macro_invocation_impl () const
   {
     return new MacroInvocation (*this);
diff --git a/gcc/rust/expand/rust-attribute-visitor.cc b/gcc/rust/expand/rust-attribute-visitor.cc
index 255e9ba..50821ed 100644
--- a/gcc/rust/expand/rust-attribute-visitor.cc
+++ b/gcc/rust/expand/rust-attribute-visitor.cc
@@ -1114,8 +1114,6 @@ AttrVisitor::visit (AST::ClosureExprInner &expr)
 void
 AttrVisitor::visit (AST::BlockExpr &expr)
 {
-  expander.push_context (MacroExpander::BLOCK);
-
   // initial strip test based on outer attrs
   expander.expand_cfg_attrs (expr.get_outer_attrs ());
   if (expander.fails_cfg_with_expand (expr.get_outer_attrs ()))
@@ -1135,31 +1133,13 @@ AttrVisitor::visit (AST::BlockExpr &expr)
       return;
     }
 
-  // strip all statements
-  auto &stmts = expr.get_statements ();
-  for (auto it = stmts.begin (); it != stmts.end ();)
-    {
-      auto &stmt = *it;
-
-      stmt->accept_vis (*this);
+  std::function<std::unique_ptr<AST::Stmt> (AST::SingleASTNode)> extractor
+    = [] (AST::SingleASTNode node) { return node.take_stmt (); };
 
-      auto fragment = expander.take_expanded_fragment (*this);
-      if (fragment.should_expand ())
-	{
-	  // Remove the current expanded invocation
-	  it = stmts.erase (it);
-	  for (auto &node : fragment.get_nodes ())
-	    {
-	      it = stmts.insert (it, node.take_stmt ());
-	      it++;
-	    }
-	}
+  expand_macro_children (MacroExpander::BLOCK, expr.get_statements (),
+			 extractor);
 
-      else if (stmt->is_marked_for_strip ())
-	it = stmts.erase (it);
-      else
-	it++;
-    }
+  expander.push_context (MacroExpander::BLOCK);
 
   // strip tail expression if exists - can actually fully remove it
   if (expr.has_tail_expr ())
@@ -2489,8 +2469,11 @@ AttrVisitor::visit (AST::Trait &trait)
   if (trait.has_where_clause ())
     expand_where_clause (trait.get_where_clause ());
 
-  // strip trait items if required
-  expand_pointer_allow_strip (trait.get_trait_items ());
+  std::function<std::unique_ptr<AST::TraitItem> (AST::SingleASTNode)> extractor
+    = [] (AST::SingleASTNode node) { return node.take_trait_item (); };
+
+  expand_macro_children (MacroExpander::TRAIT, trait.get_trait_items (),
+			 extractor);
 }
 void
 AttrVisitor::visit (AST::InherentImpl &impl)
@@ -2523,8 +2506,11 @@ AttrVisitor::visit (AST::InherentImpl &impl)
   if (impl.has_where_clause ())
     expand_where_clause (impl.get_where_clause ());
 
-  // strip inherent impl items if required
-  expand_pointer_allow_strip (impl.get_impl_items ());
+  std::function<std::unique_ptr<AST::InherentImplItem> (AST::SingleASTNode)>
+    extractor = [] (AST::SingleASTNode node) { return node.take_impl_item (); };
+
+  expand_macro_children (MacroExpander::IMPL, impl.get_impl_items (),
+			 extractor);
 }
 void
 AttrVisitor::visit (AST::TraitImpl &impl)
@@ -2659,8 +2645,12 @@ AttrVisitor::visit (AST::ExternBlock &block)
       return;
     }
 
-  // strip external items if required
-  expand_pointer_allow_strip (block.get_extern_items ());
+  std::function<std::unique_ptr<AST::ExternalItem> (AST::SingleASTNode)>
+    extractor
+    = [] (AST::SingleASTNode node) { return node.take_external_item (); };
+
+  expand_macro_children (MacroExpander::EXTERN, block.get_extern_items (),
+			 extractor);
 }
 
 // I don't think it would be possible to strip macros without expansion
diff --git a/gcc/rust/expand/rust-attribute-visitor.h b/gcc/rust/expand/rust-attribute-visitor.h
index f6327d6..6da6583 100644
--- a/gcc/rust/expand/rust-attribute-visitor.h
+++ b/gcc/rust/expand/rust-attribute-visitor.h
@@ -40,8 +40,24 @@ public:
   void expand_trait_function_decl (AST::TraitFunctionDecl &decl);
   void expand_trait_method_decl (AST::TraitMethodDecl &decl);
 
-  template <typename T> void expand_pointer_allow_strip (T &values)
+  /**
+   * Expand a set of values, erasing them if they are marked for strip, and
+   * replacing them with expanded macro nodes if necessary.
+   * This function is slightly different from `expand_pointer_allow_strip` as
+   * it can only be called in certain expansion contexts - where macro
+   * invocations are allowed.
+   *
+   * @param ctx Context to use for macro expansion
+   * @param values Iterable reference over values to replace or erase
+   * @param extractor Function to call when replacing values with the content
+   * 		of an expanded AST node
+   */
+  template <typename T, typename U>
+  void expand_macro_children (MacroExpander::ContextType ctx, T &values,
+			      std::function<U (AST::SingleASTNode)> extractor)
   {
+    expander.push_context (ctx);
+
     for (auto it = values.begin (); it != values.end ();)
       {
 	auto &value = *it;
@@ -49,10 +65,45 @@ public:
 	// mark for stripping if required
 	value->accept_vis (*this);
 
+	auto fragment = expander.take_expanded_fragment (*this);
+	if (fragment.should_expand ())
+	  {
+	    it = values.erase (it);
+	    for (auto &node : fragment.get_nodes ())
+	      {
+		it = values.insert (it, extractor (node));
+		it++;
+	      }
+	  }
+	else if (value->is_marked_for_strip ())
+	  {
+	    it = values.erase (it);
+	  }
+	else
+	  {
+	    ++it;
+	  }
+      }
+
+    expander.pop_context ();
+  }
+
+  template <typename T> void expand_pointer_allow_strip (T &values)
+  {
+    for (auto it = values.begin (); it != values.end ();)
+      {
+	auto &value = *it;
+
+	// mark for stripping if required
+	value->accept_vis (*this);
 	if (value->is_marked_for_strip ())
-	  it = values.erase (it);
+	  {
+	    it = values.erase (it);
+	  }
 	else
-	  ++it;
+	  {
+	    ++it;
+	  }
       }
   }
 
diff --git a/gcc/rust/expand/rust-macro-expand.cc b/gcc/rust/expand/rust-macro-expand.cc
index 3389f20..80f822b 100644
--- a/gcc/rust/expand/rust-macro-expand.cc
+++ b/gcc/rust/expand/rust-macro-expand.cc
@@ -786,7 +786,6 @@ parse_many (Parser<MacroInvocLexer> &parser, TokenId &delimiter,
 	    std::function<AST::SingleASTNode ()> parse_fn)
 {
   std::vector<AST::SingleASTNode> nodes;
-
   while (true)
     {
       if (parser.peek_current_token ()->get_id () == delimiter)
@@ -815,6 +814,52 @@ transcribe_many_items (Parser<MacroInvocLexer> &parser, TokenId &delimiter)
 }
 
 /**
+ * Transcribe 0 or more external items from a macro invocation
+ *
+ * @param parser Parser to extract items from
+ * @param delimiter Id of the token on which parsing should stop
+ */
+static std::vector<AST::SingleASTNode>
+transcribe_many_ext (Parser<MacroInvocLexer> &parser, TokenId &delimiter)
+{
+  return parse_many (parser, delimiter, [&parser] () {
+    auto item = parser.parse_external_item ();
+    return AST::SingleASTNode (std::move (item));
+  });
+}
+
+/**
+ * Transcribe 0 or more trait items from a macro invocation
+ *
+ * @param parser Parser to extract items from
+ * @param delimiter Id of the token on which parsing should stop
+ */
+static std::vector<AST::SingleASTNode>
+transcribe_many_trait_items (Parser<MacroInvocLexer> &parser,
+			     TokenId &delimiter)
+{
+  return parse_many (parser, delimiter, [&parser] () {
+    auto item = parser.parse_trait_item ();
+    return AST::SingleASTNode (std::move (item));
+  });
+}
+
+/**
+ * Transcribe 0 or more impl items from a macro invocation
+ *
+ * @param parser Parser to extract items from
+ * @param delimiter Id of the token on which parsing should stop
+ */
+static std::vector<AST::SingleASTNode>
+transcribe_many_impl_items (Parser<MacroInvocLexer> &parser, TokenId &delimiter)
+{
+  return parse_many (parser, delimiter, [&parser] () {
+    auto item = parser.parse_inherent_impl_item ();
+    return AST::SingleASTNode (std::move (item));
+  });
+}
+
+/**
  * Transcribe 0 or more statements from a macro invocation
  *
  * @param parser Parser to extract statements from
@@ -845,6 +890,57 @@ transcribe_expression (Parser<MacroInvocLexer> &parser)
   return {AST::SingleASTNode (std::move (expr))};
 }
 
+static std::vector<AST::SingleASTNode>
+transcribe_on_delimiter (Parser<MacroInvocLexer> &parser, bool semicolon,
+			 AST::DelimType delimiter, TokenId last_token_id)
+{
+  if (semicolon || delimiter == AST::DelimType::CURLY)
+    return transcribe_many_stmts (parser, last_token_id);
+  else
+    return transcribe_expression (parser);
+} // namespace Rust
+
+static std::vector<AST::SingleASTNode>
+transcribe_context (MacroExpander::ContextType ctx,
+		    Parser<MacroInvocLexer> &parser, bool semicolon,
+		    AST::DelimType delimiter, TokenId last_token_id)
+{
+  // The flow-chart in order to choose a parsing function is as follows:
+  //
+  // [switch special context]
+  //     -- Item --> parser.parse_item();
+  //     -- Trait --> parser.parse_trait_item();
+  //     -- Impl --> parser.parse_impl_item();
+  //     -- Extern --> parser.parse_extern_item();
+  //     -- None --> [has semicolon?]
+  //                 -- Yes --> parser.parse_stmt();
+  //                 -- No --> [switch invocation.delimiter()]
+  //                             -- { } --> parser.parse_stmt();
+  //                             -- _ --> parser.parse_expr(); // once!
+
+  // If there is a semicolon OR we are expanding a MacroInvocationSemi, then
+  // we can parse multiple items. Otherwise, parse *one* expression
+
+  switch (ctx)
+    {
+    case MacroExpander::ContextType::ITEM:
+      return transcribe_many_items (parser, last_token_id);
+      break;
+    case MacroExpander::ContextType::TRAIT:
+      return transcribe_many_trait_items (parser, last_token_id);
+      break;
+    case MacroExpander::ContextType::IMPL:
+      return transcribe_many_impl_items (parser, last_token_id);
+      break;
+    case MacroExpander::ContextType::EXTERN:
+      return transcribe_many_ext (parser, last_token_id);
+      break;
+    default:
+      return transcribe_on_delimiter (parser, semicolon, delimiter,
+				      last_token_id);
+    }
+}
+
 AST::ASTFragment
 MacroExpander::transcribe_rule (
   AST::MacroRule &match_rule, AST::DelimTokenTree &invoc_token_tree,
@@ -864,18 +960,15 @@ MacroExpander::transcribe_rule (
   std::vector<std::unique_ptr<AST::Token>> substituted_tokens
     = substitute_context.substitute_tokens ();
 
+  // handy for debugging
+  // for (auto &tok : substituted_tokens)
+  //   rust_debug ("[tok] %s", tok->as_string ().c_str ());
+
   // parse it to an ASTFragment
   MacroInvocLexer lex (std::move (substituted_tokens));
   Parser<MacroInvocLexer> parser (std::move (lex));
 
-  // handy for debugging
-  // for (auto &tok : substituted_tokens)
-  //   {
-  //     rust_debug ("tok: [%s]", tok->as_string ().c_str ());
-  //   }
-
   auto last_token_id = TokenId::RIGHT_CURLY;
-  std::vector<AST::SingleASTNode> nodes;
 
   // this is used so we can check that we delimit the stream correctly.
   switch (transcribe_tree.get_delim_type ())
@@ -905,33 +998,9 @@ MacroExpander::transcribe_rule (
   //   as a statement (either via ExpressionStatement or
   //   MacroInvocationWithSemi)
 
-  // The flow-chart in order to choose a parsing function is as follows:
-  //
-  // [is in item context?]
-  //     -- Yes --> parser.parse_item();
-  //     -- No --> [has semicolon?]
-  //                 -- Yes --> parser.parse_stmt();
-  //                 -- No --> [switch invocation.delimiter()]
-  //                             -- { } --> parser.parse_stmt();
-  //                             -- _ --> parser.parse_expr();
-
-  // If there is a semicolon OR we are expanding a MacroInvocationSemi, then
-  // we can parse multiple items. Otherwise, parse *one* expression
-
-  if (ctx == ContextType::ITEM)
-    nodes = transcribe_many_items (parser, last_token_id);
-  else if (semicolon)
-    nodes = transcribe_many_stmts (parser, last_token_id);
-  else
-    switch (invoc_token_tree.get_delim_type ())
-      {
-      case AST::CURLY:
-	nodes = transcribe_many_stmts (parser, last_token_id);
-	break;
-      default:
-	nodes = transcribe_expression (parser);
-	break;
-      }
+  auto nodes
+    = transcribe_context (ctx, parser, semicolon,
+			  invoc_token_tree.get_delim_type (), last_token_id);
 
   // emit any errors
   if (parser.has_errors ())
diff --git a/gcc/rust/expand/rust-macro-expand.h b/gcc/rust/expand/rust-macro-expand.h
index 0d0282e..61b69e4 100644
--- a/gcc/rust/expand/rust-macro-expand.h
+++ b/gcc/rust/expand/rust-macro-expand.h
@@ -187,6 +187,9 @@ struct MacroExpander
   {
     ITEM,
     BLOCK,
+    TRAIT,
+    IMPL,
+    EXTERN,
   };
 
   ExpansionCfg cfg;
diff --git a/gcc/rust/parse/rust-parse-impl.h b/gcc/rust/parse/rust-parse-impl.h
index 0bbd8fb..82e7e24 100644
--- a/gcc/rust/parse/rust-parse-impl.h
+++ b/gcc/rust/parse/rust-parse-impl.h
@@ -5251,12 +5251,11 @@ Parser<ManagedTokenSource>::parse_inherent_impl_item ()
   switch (t->get_id ())
     {
     case IDENTIFIER:
+      // FIXME: Arthur: Do we need to some lookahead here?
+      return parse_macro_invocation_semi (outer_attrs);
     case SUPER:
     case SELF:
     case CRATE:
-    case DOLLAR_SIGN:
-      // these seem to be SimplePath tokens, so this is a macro invocation semi
-      return parse_macro_invocation_semi (std::move (outer_attrs));
       case PUB: {
 	// visibility, so not a macro invocation semi - must be constant,
 	// function, or method
@@ -5813,6 +5812,8 @@ Parser<ManagedTokenSource>::parse_external_item ()
   const_TokenPtr t = lexer.peek_token ();
   switch (t->get_id ())
     {
+    case IDENTIFIER:
+      return parse_macro_invocation_semi (outer_attrs);
       case STATIC_TOK: {
 	// parse extern static item
 	lexer.skip_token ();
diff --git a/gcc/rust/parse/rust-parse.h b/gcc/rust/parse/rust-parse.h
index 5fcb305..bb5bf3d 100644
--- a/gcc/rust/parse/rust-parse.h
+++ b/gcc/rust/parse/rust-parse.h
@@ -135,6 +135,9 @@ public:
    */
   std::unique_ptr<AST::Stmt> parse_stmt (bool allow_no_semi = false);
   std::unique_ptr<AST::Type> parse_type ();
+  std::unique_ptr<AST::ExternalItem> parse_external_item ();
+  std::unique_ptr<AST::TraitItem> parse_trait_item ();
+  std::unique_ptr<AST::InherentImplItem> parse_inherent_impl_item ();
   AST::PathInExpression parse_path_in_expression ();
   std::vector<std::unique_ptr<AST::LifetimeParam> > parse_lifetime_params ();
   AST::Visibility parse_visibility ();
@@ -285,7 +288,6 @@ private:
 						      AST::AttrVec outer_attrs);
   std::unique_ptr<AST::Trait> parse_trait (AST::Visibility vis,
 					   AST::AttrVec outer_attrs);
-  std::unique_ptr<AST::TraitItem> parse_trait_item ();
   std::unique_ptr<AST::TraitItemType>
   parse_trait_type (AST::AttrVec outer_attrs);
   std::unique_ptr<AST::TraitItemConst>
@@ -293,7 +295,6 @@ private:
   AST::SelfParam parse_self_param ();
   std::unique_ptr<AST::Impl> parse_impl (AST::Visibility vis,
 					 AST::AttrVec outer_attrs);
-  std::unique_ptr<AST::InherentImplItem> parse_inherent_impl_item ();
   std::unique_ptr<AST::InherentImplItem>
   parse_inherent_impl_function_or_method (AST::Visibility vis,
 					  AST::AttrVec outer_attrs);
@@ -303,7 +304,6 @@ private:
 				       AST::AttrVec outer_attrs);
   std::unique_ptr<AST::ExternBlock>
   parse_extern_block (AST::Visibility vis, AST::AttrVec outer_attrs);
-  std::unique_ptr<AST::ExternalItem> parse_external_item ();
   AST::NamedFunctionParam parse_named_function_param (AST::AttrVec outer_attrs
 						      = AST::AttrVec ());
   AST::Method parse_method ();