Merge #938

938: First pass at declarative macro expansion  r=philberty a=philberty

This does not support repetition matchers but it supports simple
declarative macros and transcribes them. The approach taken here is that
we reuse our existing parser to call the apropriate functions as specified
as part of the MacroFragmentType enum if the parser does not have errors
parsing that item then it must be a match.
    
Then once we match a rule we have a map of the token begin/end offsets
for each fragment match, this is then used to adjust and create a new token
stream for the macro rule definition so that when we feed it to the parser
the tokens are already substituted. The resulting expression or item is
then attached to the respective macro invocation and this is then name
resolved and used for hir lowering.
    
Fixes #17 #22
Addresses #573

Co-authored-by: Philip Herron <philip.herron@embecosm.com>

1 files changed, 144 insertions, 4 deletions

diff --git a/gcc/rust/expand/rust-macro-expand.h b/gcc/rust/expand/rust-macro-expand.h
index cd9165a..d8a2d50 100644
--- a/gcc/rust/expand/rust-macro-expand.h
+++ b/gcc/rust/expand/rust-macro-expand.h
@@ -19,8 +19,13 @@
 #ifndef RUST_MACRO_EXPAND_H
 #define RUST_MACRO_EXPAND_H
 
+#include "rust-buffered-queue.h"
+#include "rust-parse.h"
+#include "rust-token.h"
 #include "rust-ast.h"
 #include "rust-macro.h"
+#include "rust-hir-map.h"
+#include "rust-name-resolver.h"
 
 // Provides objects and method prototypes for macro expansion
 
@@ -43,14 +48,101 @@ struct ExpansionCfg
   std::string crate_name = "";
 };
 
+class MacroInvocLexer
+{
+public:
+  MacroInvocLexer (std::vector<std::unique_ptr<AST::Token>> stream)
+    : offs (0), token_stream (std::move (stream))
+  {}
+
+  // Returns token n tokens ahead of current position.
+  const_TokenPtr peek_token (int n)
+  {
+    if ((offs + n) >= token_stream.size ())
+      return Token::make (END_OF_FILE, Location ());
+
+    return token_stream.at (offs + n)->get_tok_ptr ();
+  }
+  // Peeks the current token.
+  const_TokenPtr peek_token () { return peek_token (0); }
+
+  // Advances current token to n + 1 tokens ahead of current position.
+  void skip_token (int n) { offs += (n + 1); }
+
+  // Skips the current token.
+  void skip_token () { skip_token (0); }
+
+  // Splits the current token into two. Intended for use with nested generics
+  // closes (i.e. T<U<X>> where >> is wrongly lexed as one token). Note that
+  // this will only work with "simple" tokens like punctuation.
+  void split_current_token (TokenId /*new_left*/, TokenId /*new_right*/)
+  {
+    // FIXME
+    gcc_unreachable ();
+  }
+
+  std::string get_filename () const
+  {
+    gcc_unreachable ();
+    return "FIXME";
+  }
+
+  size_t get_offs () const { return offs; }
+
+private:
+  size_t offs;
+  std::vector<std::unique_ptr<AST::Token>> token_stream;
+};
+
+struct MatchedFragment
+{
+  std::string fragment_ident;
+  size_t token_offset_begin;
+  size_t token_offset_end;
+
+  std::string as_string () const
+  {
+    return fragment_ident + "=" + std::to_string (token_offset_begin) + ":"
+	   + std::to_string (token_offset_end);
+  }
+};
+
+class SubstitutionScope
+{
+public:
+  SubstitutionScope () : stack () {}
+
+  void push () { stack.push_back ({}); }
+
+  std::map<std::string, MatchedFragment> pop ()
+  {
+    auto top = stack.back ();
+    stack.pop_back ();
+    return top;
+  }
+
+  std::map<std::string, MatchedFragment> &peek () { return stack.back (); }
+
+private:
+  std::vector<std::map<std::string, MatchedFragment>> stack;
+};
+
 // Object used to store shared data (between functions) for macro expansion.
 struct MacroExpander
 {
+  enum ContextType
+  {
+    ITEM,
+    BLOCK,
+  };
+
   ExpansionCfg cfg;
   unsigned int expansion_depth = 0;
 
   MacroExpander (AST::Crate &crate, ExpansionCfg cfg, Session &session)
-    : cfg (cfg), crate (crate), session (session)
+    : cfg (cfg), crate (crate), session (session),
+      sub_stack (SubstitutionScope ()), resolver (Resolver::Resolver::get ()),
+      mappings (Analysis::Mappings::get ())
   {}
 
   ~MacroExpander () = default;
@@ -61,11 +153,14 @@ struct MacroExpander
   /* Expands a macro invocation (not macro invocation semi) - possibly make both
    * have similar duck-typed interface and use templates?*/
   // should this be public or private?
-  void expand_invoc (std::unique_ptr<AST::MacroInvocation> &invoc);
+  void expand_invoc (AST::MacroInvocation &invoc);
+  void expand_invoc_semi (AST::MacroInvocationSemi &invoc);
 
   // Expands a single declarative macro.
-  AST::ASTFragment expand_decl_macro (AST::MacroInvocData &invoc,
-				      AST::MacroRulesDefinition &rules_def);
+  AST::ASTFragment expand_decl_macro (Location locus,
+				      AST::MacroInvocData &invoc,
+				      AST::MacroRulesDefinition &rules_def,
+				      bool semicolon);
 
   void expand_cfg_attrs (AST::AttrVec &attrs);
   bool fails_cfg (const AST::AttrVec &attr) const;
@@ -76,10 +171,55 @@ struct MacroExpander
   // Get the literal representation of a cfg! macro.
   AST::Literal expand_cfg_macro (AST::MacroInvocData &invoc);
 
+  bool depth_exceeds_recursion_limit () const;
+
+  bool try_match_rule (AST::MacroRule &match_rule,
+		       AST::DelimTokenTree &invoc_token_tree);
+
+  AST::ASTFragment
+  transcribe_rule (AST::MacroRule &match_rule,
+		   AST::DelimTokenTree &invoc_token_tree,
+		   std::map<std::string, MatchedFragment> &matched_fragments,
+		   bool semicolon, ContextType ctx);
+
+  bool match_fragment (Parser<MacroInvocLexer> &parser,
+		       AST::MacroMatchFragment &fragment);
+
+  bool match_token (Parser<MacroInvocLexer> &parser, AST::Token &token);
+
+  bool match_repetition (Parser<MacroInvocLexer> &parser,
+			 AST::MacroMatchRepetition &rep);
+
+  bool match_matcher (Parser<MacroInvocLexer> &parser,
+		      AST::MacroMatcher &matcher);
+
+  static std::vector<std::unique_ptr<AST::Token>>
+  substitute_tokens (std::vector<std::unique_ptr<AST::Token>> &input,
+		     std::vector<std::unique_ptr<AST::Token>> &macro,
+		     std::map<std::string, MatchedFragment> &fragments);
+
+  void push_context (ContextType t) { context.push_back (t); }
+
+  ContextType pop_context ()
+  {
+    ContextType t = context.back ();
+    context.pop_back ();
+    return t;
+  }
+
+  ContextType peek_context () { return context.back (); }
+
 private:
   AST::Crate &crate;
   Session &session;
+  SubstitutionScope sub_stack;
+  std::vector<ContextType> context;
+
+public:
+  Resolver::Resolver *resolver;
+  Analysis::Mappings *mappings;
 };
+
 } // namespace Rust
 
 #endif