Merge #938

938: First pass at declarative macro expansion  r=philberty a=philberty

This does not support repetition matchers but it supports simple
declarative macros and transcribes them. The approach taken here is that
we reuse our existing parser to call the apropriate functions as specified
as part of the MacroFragmentType enum if the parser does not have errors
parsing that item then it must be a match.
    
Then once we match a rule we have a map of the token begin/end offsets
for each fragment match, this is then used to adjust and create a new token
stream for the macro rule definition so that when we feed it to the parser
the tokens are already substituted. The resulting expression or item is
then attached to the respective macro invocation and this is then name
resolved and used for hir lowering.
    
Fixes #17 #22
Addresses #573

Co-authored-by: Philip Herron <philip.herron@embecosm.com>

3 files changed, 55 insertions, 43 deletions

diff --git a/gcc/rust/parse/rust-parse-impl.h b/gcc/rust/parse/rust-parse-impl.h
index 7483818..784e6d1 100644
--- a/gcc/rust/parse/rust-parse-impl.h
+++ b/gcc/rust/parse/rust-parse-impl.h
@@ -905,6 +905,9 @@ Parser<ManagedTokenSource>::parse_delim_token_tree ()
 
   // parse actual token tree vector - 0 or more
   std::vector<std::unique_ptr<AST::TokenTree>> token_trees_in_tree;
+  auto delim_open
+    = std::unique_ptr<AST::Token> (new AST::Token (std::move (t)));
+  token_trees_in_tree.push_back (std::move (delim_open));
 
   // repeat loop until finding the matching delimiter
   t = lexer.peek_token ();
@@ -929,6 +932,9 @@ Parser<ManagedTokenSource>::parse_delim_token_tree ()
       // lexer.skip_token();
       t = lexer.peek_token ();
     }
+  auto delim_close
+    = std::unique_ptr<AST::Token> (new AST::Token (std::move (t)));
+  token_trees_in_tree.push_back (std::move (delim_close));
 
   AST::DelimTokenTree token_tree (delim_type, std::move (token_trees_in_tree),
 				  initial_loc);
@@ -1565,6 +1571,9 @@ Parser<ManagedTokenSource>::parse_macro_invocation_semi (
 
   // parse actual token trees
   std::vector<std::unique_ptr<AST::TokenTree>> token_trees;
+  auto delim_open
+    = std::unique_ptr<AST::Token> (new AST::Token (std::move (t)));
+  token_trees.push_back (std::move (delim_open));
 
   t = lexer.peek_token ();
   // parse token trees until the initial delimiter token is found again
@@ -1587,6 +1596,9 @@ Parser<ManagedTokenSource>::parse_macro_invocation_semi (
 
       t = lexer.peek_token ();
     }
+  auto delim_close
+    = std::unique_ptr<AST::Token> (new AST::Token (std::move (t)));
+  token_trees.push_back (std::move (delim_close));
 
   AST::DelimTokenTree delim_tok_tree (delim_type, std::move (token_trees),
 				      tok_tree_locus);
@@ -1605,6 +1617,7 @@ Parser<ManagedTokenSource>::parse_macro_invocation_semi (
 	  if (!skip_token (SEMICOLON))
 	    {
 	      // as this is the end, allow recovery (probably) - may change
+
 	      return std::unique_ptr<AST::MacroInvocationSemi> (
 		new AST::MacroInvocationSemi (std::move (invoc_data),
 					      std::move (outer_attrs),
@@ -11755,6 +11768,9 @@ Parser<ManagedTokenSource>::parse_path_based_stmt_or_expr (
 
 	// parse actual token trees
 	std::vector<std::unique_ptr<AST::TokenTree>> token_trees;
+	auto delim_open
+	  = std::unique_ptr<AST::Token> (new AST::Token (std::move (t3)));
+	token_trees.push_back (std::move (delim_open));
 
 	t3 = lexer.peek_token ();
 	// parse token trees until the initial delimiter token is found again
@@ -11779,6 +11795,10 @@ Parser<ManagedTokenSource>::parse_path_based_stmt_or_expr (
 	    t3 = lexer.peek_token ();
 	  }
 
+	auto delim_close
+	  = std::unique_ptr<AST::Token> (new AST::Token (std::move (t3)));
+	token_trees.push_back (std::move (delim_close));
+
 	// parse end delimiters
 	t3 = lexer.peek_token ();
 	if (token_id_matches_delims (t3->get_id (), type))
@@ -12070,6 +12090,9 @@ Parser<ManagedTokenSource>::parse_macro_invocation_maybe_semi (
 
   // parse actual token trees
   std::vector<std::unique_ptr<AST::TokenTree>> token_trees;
+  auto delim_open
+    = std::unique_ptr<AST::Token> (new AST::Token (std::move (t3)));
+  token_trees.push_back (std::move (delim_open));
 
   t3 = lexer.peek_token ();
   // parse token trees until the initial delimiter token is found again
@@ -12092,6 +12115,9 @@ Parser<ManagedTokenSource>::parse_macro_invocation_maybe_semi (
 
       t3 = lexer.peek_token ();
     }
+  auto delim_close
+    = std::unique_ptr<AST::Token> (new AST::Token (std::move (t3)));
+  token_trees.push_back (std::move (delim_close));
 
   // parse end delimiters
   t3 = lexer.peek_token ();
diff --git a/gcc/rust/parse/rust-parse.cc b/gcc/rust/parse/rust-parse.cc
index f2c1301..f995e4b 100644
--- a/gcc/rust/parse/rust-parse.cc
+++ b/gcc/rust/parse/rust-parse.cc
@@ -18,32 +18,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "rust-linemap.h"
 #include "rust-diagnostics.h"
 
-#if 0
-#include "config.h"
-#include "system.h"
-#include "coretypes.h"
-#include "target.h"
-#include "tree.h"
-#include "tree-iterator.h"
-#include "input.h"
-#include "diagnostic.h"
-#include "stringpool.h"
-#include "cgraph.h"
-#include "gimplify.h"
-#include "gimple-expr.h"
-#include "convert.h"
-#include "print-tree.h"
-#include "stor-layout.h"
-#include "fold-const.h"
-/* order: config, system, coretypes, target, tree, tree-iterator, input, diagnostic, stringpool,
- * cgraph, gimplify, gimple-expr, convert, print-tree, stor-layout, fold-const  */
-// probably don't need all these
-#endif
-// maybe put these back in if compiling no longer works
-
-/* TODO: move non-essential stuff back here from rust-parse-impl.h after
- * confirming that it works */
-
 namespace Rust {
 
 std::string
diff --git a/gcc/rust/parse/rust-parse.h b/gcc/rust/parse/rust-parse.h
index acab7ff..5ee7b4e 100644
--- a/gcc/rust/parse/rust-parse.h
+++ b/gcc/rust/parse/rust-parse.h
@@ -86,6 +86,29 @@ struct ParseRestrictions
 // TODO: if updated to C++20, ManagedTokenSource would be useful as a concept
 template <typename ManagedTokenSource> class Parser
 {
+public:
+  bool skip_token (TokenId t);
+
+  std::unique_ptr<AST::Expr>
+  parse_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
+	      ParseRestrictions restrictions = ParseRestrictions ());
+
+  std::unique_ptr<AST::LiteralExpr> parse_literal_expr (AST::AttrVec outer_attrs
+							= AST::AttrVec ());
+
+  std::unique_ptr<AST::BlockExpr>
+  parse_block_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
+		    Location pratt_parsed_loc = Linemap::unknown_location ());
+
+  std::unique_ptr<AST::Item> parse_item (bool called_from_statement);
+  std::unique_ptr<AST::Pattern> parse_pattern ();
+  std::unique_ptr<AST::Stmt> parse_stmt ();
+  std::unique_ptr<AST::Type> parse_type ();
+  AST::PathInExpression parse_path_in_expression ();
+  std::vector<std::unique_ptr<AST::LifetimeParam> > parse_lifetime_params ();
+  AST::Visibility parse_visibility ();
+  std::unique_ptr<AST::IdentifierPattern> parse_identifier_pattern ();
+
 private:
   void skip_after_semicolon ();
   void skip_after_end ();
@@ -93,7 +116,6 @@ private:
   void skip_after_next_block ();
   void skip_after_end_attribute ();
 
-  bool skip_token (TokenId t);
   const_TokenPtr expect_token (TokenId t);
   void unexpected_token (const_TokenPtr t);
   bool skip_generics_right_angle ();
@@ -118,7 +140,6 @@ private:
   AST::GenericArgs parse_path_generic_args ();
   AST::GenericArgsBinding parse_generic_args_binding ();
   AST::TypePathFunction parse_type_path_function ();
-  AST::PathInExpression parse_path_in_expression ();
   AST::PathExprSegment parse_path_expr_segment ();
   AST::QualifiedPathInExpression
   // When given a pratt_parsed_loc, use it as the location of the
@@ -147,10 +168,8 @@ private:
   std::unique_ptr<AST::MacroMatchRepetition> parse_macro_match_repetition ();
 
   // Top-level item-related
-  std::unique_ptr<AST::Item> parse_item (bool called_from_statement);
   std::unique_ptr<AST::VisItem> parse_vis_item (AST::AttrVec outer_attrs);
   std::unique_ptr<AST::MacroItem> parse_macro_item (AST::AttrVec outer_attrs);
-  AST::Visibility parse_visibility ();
 
   // VisItem subclass-related
   std::unique_ptr<AST::Module> parse_module (AST::Visibility vis,
@@ -169,7 +188,7 @@ private:
   template <typename EndTokenPred>
   std::vector<std::unique_ptr<AST::GenericParam> >
   parse_generic_params (EndTokenPred is_end_token);
-  std::vector<std::unique_ptr<AST::LifetimeParam> > parse_lifetime_params ();
+
   template <typename EndTokenPred>
   std::vector<std::unique_ptr<AST::LifetimeParam> >
   parse_lifetime_params (EndTokenPred is_end_token);
@@ -260,9 +279,6 @@ private:
 
   // Expression-related (Pratt parsed)
   std::unique_ptr<AST::Expr>
-  parse_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
-	      ParseRestrictions restrictions = ParseRestrictions ());
-  std::unique_ptr<AST::Expr>
   parse_expr (int right_binding_power,
 	      AST::AttrVec outer_attrs = AST::AttrVec (),
 	      ParseRestrictions restrictions = ParseRestrictions ());
@@ -478,9 +494,6 @@ private:
   // When given a pratt_parsed_loc, use it as the location of the
   // first token parsed in the expression (the parsing of that first
   // token should be skipped).
-  std::unique_ptr<AST::BlockExpr>
-  parse_block_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
-		    Location pratt_parsed_loc = Linemap::unknown_location ());
   std::unique_ptr<AST::IfExpr>
   parse_if_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
 		 Location pratt_parsed_loc = Linemap::unknown_location ());
@@ -518,8 +531,7 @@ private:
   std::unique_ptr<AST::ClosureExpr> parse_closure_expr (AST::AttrVec outer_attrs
 							= AST::AttrVec ());
   AST::ClosureParam parse_closure_param ();
-  std::unique_ptr<AST::LiteralExpr> parse_literal_expr (AST::AttrVec outer_attrs
-							= AST::AttrVec ());
+
   // When given a pratt_parsed_loc, use it as the location of the
   // first token parsed in the expression (the parsing of that first
   // token should be skipped).
@@ -548,7 +560,6 @@ private:
   bool will_be_expr_with_block ();
 
   // Type-related
-  std::unique_ptr<AST::Type> parse_type ();
   std::unique_ptr<AST::TypeNoBounds> parse_type_no_bounds ();
   std::unique_ptr<AST::TypeNoBounds> parse_slice_or_array_type ();
   std::unique_ptr<AST::RawPointerType> parse_raw_pointer_type ();
@@ -561,7 +572,6 @@ private:
   AST::MaybeNamedParam parse_maybe_named_param (AST::AttrVec outer_attrs);
 
   // Statement-related
-  std::unique_ptr<AST::Stmt> parse_stmt ();
   std::unique_ptr<AST::LetStmt> parse_let_stmt (AST::AttrVec outer_attrs);
   std::unique_ptr<AST::ExprStmt> parse_expr_stmt (AST::AttrVec outer_attrs);
   std::unique_ptr<AST::ExprStmtWithBlock>
@@ -574,13 +584,11 @@ private:
   ExprOrStmt parse_path_based_stmt_or_expr (AST::AttrVec outer_attrs);
 
   // Pattern-related
-  std::unique_ptr<AST::Pattern> parse_pattern ();
   std::unique_ptr<AST::Pattern> parse_literal_or_range_pattern ();
   std::unique_ptr<AST::RangePatternBound> parse_range_pattern_bound ();
   std::unique_ptr<AST::ReferencePattern> parse_reference_pattern ();
   std::unique_ptr<AST::Pattern> parse_grouped_or_tuple_pattern ();
   std::unique_ptr<AST::SlicePattern> parse_slice_pattern ();
-  std::unique_ptr<AST::IdentifierPattern> parse_identifier_pattern ();
   std::unique_ptr<AST::Pattern> parse_ident_leading_pattern ();
   std::unique_ptr<AST::TupleStructItems> parse_tuple_struct_items ();
   AST::StructPatternElements parse_struct_pattern_elems ();
@@ -617,6 +625,10 @@ public:
   // Get a reference to the list of errors encountered
   std::vector<Error> &get_errors () { return error_table; }
 
+  const ManagedTokenSource &get_token_source () const { return lexer; }
+
+  const_TokenPtr peek_current_token () { return lexer.peek_token (0); }
+
 private:
   // The token source (usually lexer) associated with the parser.
   ManagedTokenSource lexer;