Add initial support for macro expansion

This is the first pass at implementing macros more testcases are needed.

This does not support repetition matchers but it supports simple
declarative macros and transcribes them. The approach taken here is that
we reuse our existing parser to call the apropriate functions as specified
as part of the MacroFragmentType enum if the parser does not have errors
parsing that item then it must be a match.

Then once we match a rule we have a map of the token begin/end offsets
for each fragment match, this is then used to adjust and create a new token
stream for the macro rule definition so that when we feed it to the parser
the tokens are already substituted. The resulting expression or item is
then attached to the respective macro invocation and this is then name
resolved and used for hir lowering.

Fixes #17 #22
Addresses #573

15 files changed, 1017 insertions, 255 deletions

diff --git a/gcc/rust/ast/rust-ast-full-test.cc b/gcc/rust/ast/rust-ast-full-test.cc
index 68f6f8c..ee7c407 100644
--- a/gcc/rust/ast/rust-ast-full-test.cc
+++ b/gcc/rust/ast/rust-ast-full-test.cc
@@ -4390,7 +4390,8 @@ DelimTokenTree::to_token_stream () const
 
   // simulate presence of delimiters
   const_TokenPtr left_paren
-    = Rust::Token::make (LEFT_PAREN, Linemap::unknown_location ());
+    = Rust::Token::make (left_delim_type_tok_token_id (delim_type),
+			 Linemap::unknown_location ());
   tokens.push_back (
     std::unique_ptr<Token> (new Token (std::move (left_paren))));
 
@@ -4403,7 +4404,8 @@ DelimTokenTree::to_token_stream () const
     }
 
   const_TokenPtr right_paren
-    = Rust::Token::make (RIGHT_PAREN, Linemap::unknown_location ());
+    = Rust::Token::make (right_delim_type_tok_token_id (delim_type),
+			 Linemap::unknown_location ());
   tokens.push_back (
     std::unique_ptr<Token> (new Token (std::move (right_paren))));
 
diff --git a/gcc/rust/ast/rust-ast.h b/gcc/rust/ast/rust-ast.h
index ba973f1..8411f65 100644
--- a/gcc/rust/ast/rust-ast.h
+++ b/gcc/rust/ast/rust-ast.h
@@ -22,9 +22,6 @@
 
 #include "rust-system.h"
 #include "rust-hir-map.h"
-
-// gccrs imports
-// required for AST::Token
 #include "rust-token.h"
 #include "rust-location.h"
 
@@ -32,7 +29,6 @@ namespace Rust {
 // TODO: remove typedefs and make actual types for these
 typedef std::string Identifier;
 typedef int TupleIndex;
-
 struct Session;
 
 namespace AST {
@@ -48,34 +44,6 @@ enum DelimType
   CURLY
 };
 
-// Base AST node object - TODO is this really required or useful? Where to draw
-// line?
-/*class Node {
-  public:
-    // Gets node's Location.
-    Location get_locus() const {
-	return loc;
-    }
-
-    // Sets node's Location.
-    void set_locus(Location loc_) {
-	loc = loc_;
-    }
-
-    // Get node output as a string. Pure virtual.
-    virtual std::string as_string() const = 0;
-
-    virtual ~Node() {}
-
-    // TODO: constructor including Location? Make all derived classes have
-Location?
-
-  private:
-    // The node's location.
-    Location loc;
-};*/
-// decided to not have node as a "node" would never need to be stored
-
 // forward decl for use in token tree method
 class Token;
 
@@ -108,6 +76,14 @@ protected:
 class MacroMatch
 {
 public:
+  enum MacroMatchType
+  {
+    Fragment,
+    Repetition,
+    Matcher,
+    Tok
+  };
+
   virtual ~MacroMatch () {}
 
   virtual std::string as_string () const = 0;
@@ -121,6 +97,8 @@ public:
 
   virtual void accept_vis (ASTVisitor &vis) = 0;
 
+  virtual MacroMatchType get_macro_match_type () const = 0;
+
 protected:
   // pure virtual clone implementation
   virtual MacroMatch *clone_macro_match_impl () const = 0;
@@ -234,6 +212,11 @@ public:
   // Get a new token pointer copy.
   const_TokenPtr get_tok_ptr () const { return tok_ref; }
 
+  MacroMatchType get_macro_match_type () const override
+  {
+    return MacroMatchType::Tok;
+  }
+
 protected:
   // No virtual for now as not polymorphic but can be in future
   /*virtual*/ Token *clone_token_impl () const { return new Token (*this); }
@@ -788,6 +771,43 @@ public:
   {
     return AttrInput::AttrInputType::TOKEN_TREE;
   }
+
+  std::vector<std::unique_ptr<TokenTree> > &get_token_trees ()
+  {
+    return token_trees;
+  }
+
+  DelimType get_delim_type () const { return delim_type; }
+
+  static TokenId left_delim_type_tok_token_id (DelimType delim_type)
+  {
+    switch (delim_type)
+      {
+      case PARENS:
+	return LEFT_PAREN;
+      case SQUARE:
+	return LEFT_SQUARE;
+      case CURLY:
+	return LEFT_CURLY;
+      default:
+	gcc_unreachable ();
+      }
+  }
+
+  static TokenId right_delim_type_tok_token_id (DelimType delim_type)
+  {
+    switch (delim_type)
+      {
+      case PARENS:
+	return RIGHT_PAREN;
+      case SQUARE:
+	return RIGHT_SQUARE;
+      case CURLY:
+	return RIGHT_CURLY;
+      default:
+	gcc_unreachable ();
+      }
+  }
 };
 
 /* Forward decl - definition moved to rust-expr.h as it requires LiteralExpr to
diff --git a/gcc/rust/ast/rust-macro.h b/gcc/rust/ast/rust-macro.h
index e0524c6..1e95ebe 100644
--- a/gcc/rust/ast/rust-macro.h
+++ b/gcc/rust/ast/rust-macro.h
@@ -24,6 +24,7 @@
 
 namespace Rust {
 namespace AST {
+
 // Decls as definitions moved to rust-ast.h
 class MacroItem;
 class MacroInvocationSemi;
@@ -83,6 +84,135 @@ get_frag_spec_from_str (std::string str)
     }
 }
 
+class SingleASTNode
+{
+public:
+  enum NodeType
+  {
+    EXPRESSION,
+    ITEM,
+  };
+
+  SingleASTNode (std::unique_ptr<Expr> expr)
+    : type (EXPRESSION), expr (std::move (expr)), item (nullptr)
+  {}
+
+  SingleASTNode (std::unique_ptr<Item> item)
+    : type (ITEM), expr (nullptr), item (std::move (item))
+  {}
+
+  SingleASTNode (SingleASTNode const &other)
+  {
+    type = other.type;
+    switch (type)
+      {
+      case EXPRESSION:
+	expr = other.expr->clone_expr ();
+	break;
+
+      case ITEM:
+	item = other.item->clone_item ();
+	break;
+      }
+  }
+
+  SingleASTNode operator= (SingleASTNode const &other)
+  {
+    type = other.type;
+    switch (type)
+      {
+      case EXPRESSION:
+	expr = other.expr->clone_expr ();
+	break;
+
+      case ITEM:
+	item = other.item->clone_item ();
+	break;
+      }
+    return *this;
+  }
+
+  SingleASTNode (SingleASTNode &&other) = default;
+  SingleASTNode &operator= (SingleASTNode &&other) = default;
+
+  std::unique_ptr<Expr> &get_expr ()
+  {
+    rust_assert (type == EXPRESSION);
+    return expr;
+  }
+
+  std::unique_ptr<Item> &get_item ()
+  {
+    rust_assert (type == ITEM);
+    return item;
+  }
+
+  void accept_vis (ASTVisitor &vis)
+  {
+    switch (type)
+      {
+      case EXPRESSION:
+	expr->accept_vis (vis);
+	break;
+
+      case ITEM:
+	item->accept_vis (vis);
+	break;
+      }
+  }
+
+private:
+  NodeType type;
+  std::unique_ptr<Expr> expr;
+  std::unique_ptr<Item> item;
+  // TODO add meta attribute
+};
+
+/* Basically, a "fragment" that can be incorporated into the AST, created as
+ * a result of macro expansion. Really annoying to work with due to the fact
+ * that macros can really expand to anything. As such, horrible representation
+ * at the moment. */
+class ASTFragment
+{
+private:
+  /* basic idea: essentially, a vector of tagged unions of different AST node
+   * types. Now, this could actually be stored without a tagged union if the
+   * different AST node types had a unified parent, but that would create
+   * issues with the diamond problem or significant performance penalties. So
+   * a tagged union had to be used instead. A vector is used to represent the
+   * ability for a macro to expand to two statements, for instance. */
+
+  std::vector<SingleASTNode> nodes;
+
+public:
+  ASTFragment (std::vector<SingleASTNode> nodes) : nodes (std::move (nodes)) {}
+
+  ASTFragment (ASTFragment const &other)
+  {
+    nodes.clear ();
+    nodes.reserve (other.nodes.size ());
+    for (auto &n : other.nodes)
+      {
+	nodes.push_back (n);
+      }
+  }
+
+  ASTFragment &operator= (ASTFragment const &other)
+  {
+    nodes.clear ();
+    nodes.reserve (other.nodes.size ());
+    for (auto &n : other.nodes)
+      {
+	nodes.push_back (n);
+      }
+    return *this;
+  }
+
+  static ASTFragment create_empty () { return ASTFragment ({}); }
+
+  std::vector<SingleASTNode> &get_nodes () { return nodes; }
+};
+
 // A macro match that has an identifier and fragment spec
 class MacroMatchFragment : public MacroMatch
 {
@@ -109,6 +239,14 @@ public:
 
   void accept_vis (ASTVisitor &vis) override;
 
+  MacroMatchType get_macro_match_type () const override
+  {
+    return MacroMatchType::Fragment;
+  }
+
+  Identifier get_ident () const { return ident; }
+  MacroFragSpec get_frag_spec () const { return frag_spec; }
+
 protected:
   /* Use covariance to implement clone function as returning this object rather
    * than base */
@@ -192,6 +330,11 @@ public:
 
   void accept_vis (ASTVisitor &vis) override;
 
+  MacroMatchType get_macro_match_type () const override
+  {
+    return MacroMatchType::Repetition;
+  }
+
 protected:
   /* Use covariance to implement clone function as returning this object rather
    * than base */
@@ -259,6 +402,14 @@ public:
 
   void accept_vis (ASTVisitor &vis) override;
 
+  MacroMatchType get_macro_match_type () const override
+  {
+    return MacroMatchType::Matcher;
+  }
+
+  DelimType get_delim_type () const { return delim_type; }
+  std::vector<std::unique_ptr<MacroMatch> > &get_matches () { return matches; }
+
 protected:
   /* Use covariance to implement clone function as returning this object rather
    * than base */
@@ -288,6 +439,8 @@ public:
   std::string as_string () const { return token_tree.as_string (); }
 
   Location get_locus () const { return locus; }
+
+  DelimTokenTree &get_token_tree () { return token_tree; }
 };
 
 // A macro rule? Matcher and transcriber pair?
@@ -319,6 +472,9 @@ public:
   Location get_locus () const { return locus; }
 
   std::string as_string () const;
+
+  MacroMatcher &get_matcher () { return matcher; }
+  MacroTranscriber &get_transcriber () { return transcriber; }
 };
 
 // A macro rules definition item AST node
@@ -365,6 +521,11 @@ public:
 
   Location get_locus () const override final { return locus; }
 
+  Identifier get_rule_name () const { return rule_name; }
+
+  std::vector<MacroRule> &get_rules () { return rules; }
+  const std::vector<MacroRule> &get_rules () const { return rules; }
+
 protected:
   /* Use covariance to implement clone function as returning this object rather
    * than base */
@@ -384,13 +545,17 @@ class MacroInvocation : public TypeNoBounds,
   MacroInvocData invoc_data;
   Location locus;
 
+  // this is the expanded macro
+  ASTFragment fragment;
+
 public:
   std::string as_string () const override;
 
   MacroInvocation (MacroInvocData invoc_data,
 		   std::vector<Attribute> outer_attrs, Location locus)
     : outer_attrs (std::move (outer_attrs)),
-      invoc_data (std::move (invoc_data)), locus (locus)
+      invoc_data (std::move (invoc_data)), locus (locus),
+      fragment (ASTFragment::create_empty ())
   {}
 
   Location get_locus () const override final { return locus; }
@@ -417,6 +582,12 @@ public:
     return ExprWithoutBlock::get_node_id ();
   }
 
+  MacroInvocData &get_invoc_data () { return invoc_data; }
+
+  ASTFragment &get_fragment () { return fragment; }
+
+  void set_fragment (ASTFragment &&f) { fragment = std::move (f); }
+
 protected:
   /* Use covariance to implement clone function as returning this object rather
    * than base */
@@ -651,62 +822,6 @@ protected:
   }
 };
 
-/* Should be a tagged union to save space but implemented as struct due to
- * technical difficulties. TODO: fix
- * Basically, a single AST node used inside an AST fragment. */
-struct SingleASTNode
-{
-  std::unique_ptr<Expr> expr;
-  std::unique_ptr<Stmt> stmt;
-  std::unique_ptr<Item> item;
-  std::unique_ptr<Type> type;
-  std::unique_ptr<Pattern> pattern;
-  std::unique_ptr<TraitItem> trait_item;
-  std::unique_ptr<InherentImplItem> inherent_impl_item;
-  std::unique_ptr<TraitImplItem> trait_impl_item;
-  std::unique_ptr<ExternalItem> external_item;
-
-  SingleASTNode (std::unique_ptr<Expr> expr) : expr (std::move (expr)) {}
-  SingleASTNode (std::unique_ptr<Stmt> stmt) : stmt (std::move (stmt)) {}
-  SingleASTNode (std::unique_ptr<Item> item) : item (std::move (item)) {}
-  SingleASTNode (std::unique_ptr<Type> type) : type (std::move (type)) {}
-  SingleASTNode (std::unique_ptr<Pattern> pattern)
-    : pattern (std::move (pattern))
-  {}
-  SingleASTNode (std::unique_ptr<TraitItem> trait_item)
-    : trait_item (std::move (trait_item))
-  {}
-  SingleASTNode (std::unique_ptr<InherentImplItem> inherent_impl_item)
-    : inherent_impl_item (std::move (inherent_impl_item))
-  {}
-  SingleASTNode (std::unique_ptr<TraitImplItem> trait_impl_item)
-    : trait_impl_item (std::move (trait_impl_item))
-  {}
-  SingleASTNode (std::unique_ptr<ExternalItem> external_item)
-    : external_item (std::move (external_item))
-  {}
-};
-
-/* Basically, a "fragment" that can be incorporated into the AST, created as
- * a result of macro expansion. Really annoying to work with due to the fact
- * that macros can really expand to anything. As such, horrible representation
- * at the moment. */
-struct ASTFragment
-{
-private:
-  /* basic idea: essentially, a vector of tagged unions of different AST node
-   * types. Now, this could actually be stored without a tagged union if the
-   * different AST node types had a unified parent, but that would create
-   * issues with the diamond problem or significant performance penalties. So
-   * a tagged union had to be used instead. A vector is used to represent the
-   * ability for a macro to expand to two statements, for instance. */
-
-  std::vector<SingleASTNode> nodes;
-
-public:
-  ASTFragment (std::vector<SingleASTNode> nodes) : nodes (std::move (nodes)) {}
-};
-
 // Object that parses macros from a token stream.
 /* TODO: would "AttributeParser" be a better name? MetaItems are only for
  * attributes, I believe */
diff --git a/gcc/rust/expand/rust-macro-expand.cc b/gcc/rust/expand/rust-macro-expand.cc
index 4a096c3..306b2e5 100644
--- a/gcc/rust/expand/rust-macro-expand.cc
+++ b/gcc/rust/expand/rust-macro-expand.cc
@@ -20,6 +20,7 @@
 #include "rust-ast-full.h"
 #include "rust-ast-visitor.h"
 #include "rust-diagnostics.h"
+#include "rust-parse.h"
 
 namespace Rust {
 // Visitor used to expand attributes.
@@ -2509,9 +2510,20 @@ public:
       }
 
     // I don't think any macro rules can be stripped in any way
+
+    auto path = Resolver::CanonicalPath::new_seg (rules_def.get_node_id (),
+						  rules_def.get_rule_name ());
+    expander.resolver->get_macro_scope ().insert (path,
+						  rules_def.get_node_id (),
+						  rules_def.get_locus ());
+    expander.mappings->insert_macro_def (&rules_def);
   }
+
   void visit (AST::MacroInvocation &macro_invoc) override
   {
+    // FIXME
+    // we probably need another recurision check here
+
     // initial strip test based on outer attrs
     expander.expand_cfg_attrs (macro_invoc.get_outer_attrs ());
     if (expander.fails_cfg_with_expand (macro_invoc.get_outer_attrs ()))
@@ -2522,8 +2534,9 @@ public:
 
     // I don't think any macro token trees can be stripped in any way
 
-    // TODO: maybe have stripping behaviour for the cfg! macro here?
+    expander.expand_invoc (macro_invoc);
   }
+
   void visit (AST::MetaItemPath &) override {}
   void visit (AST::MetaItemSeq &) override {}
   void visit (AST::MetaWord &) override {}
@@ -3011,7 +3024,7 @@ MacroExpander::parse_macro_to_meta_item (AST::MacroInvocData &invoc)
     }
   else
     {
-      std::vector<std::unique_ptr<AST::MetaItemInner> > meta_items (
+      std::vector<std::unique_ptr<AST::MetaItemInner>> meta_items (
 	std::move (converted_input->get_items ()));
       invoc.set_meta_item_output (std::move (meta_items));
     }
@@ -3038,9 +3051,9 @@ MacroExpander::expand_cfg_macro (AST::MacroInvocData &invoc)
     return AST::Literal ("false", AST::Literal::BOOL, CORETYPE_BOOL);
 }
 
-#if 0
 AST::ASTFragment
-MacroExpander::expand_decl_macro (AST::MacroInvocData &invoc,
+MacroExpander::expand_decl_macro (Location invoc_locus,
+				  AST::MacroInvocData &invoc,
 				  AST::MacroRulesDefinition &rules_def)
 {
   // ensure that both invocation and rules are in a valid state
@@ -3081,49 +3094,87 @@ MacroExpander::expand_decl_macro (AST::MacroInvocData &invoc,
    * TokenTree). This will prevent re-conversion of Tokens between each type
    * all the time, while still allowing the heterogenous storage of token trees.
    */
+
+  AST::DelimTokenTree &invoc_token_tree = invoc.get_delim_tok_tree ();
+
+  // find matching arm
+  AST::MacroRule *matched_rule = nullptr;
+  std::map<std::string, MatchedFragment> matched_fragments;
+  for (auto &rule : rules_def.get_rules ())
+    {
+      sub_stack.push ();
+      bool did_match_rule = try_match_rule (rule, invoc_token_tree);
+      matched_fragments = sub_stack.pop ();
+
+      if (did_match_rule)
+	{
+	  matched_rule = &rule;
+	  break;
+	}
+    }
+
+  if (matched_rule == nullptr)
+    {
+      RichLocation r (invoc_locus);
+      r.add_range (rules_def.get_locus ());
+      rust_error_at (r, "Failed to match any rule within macro");
+      return AST::ASTFragment::create_empty ();
+    }
+
+  return transcribe_rule (*matched_rule, invoc_token_tree, matched_fragments);
 }
-#endif
 
 void
-MacroExpander::expand_invoc (std::unique_ptr<AST::MacroInvocation> &invoc)
+MacroExpander::expand_invoc (AST::MacroInvocation &invoc)
 {
-  /* if current expansion depth > recursion limit, create an error (maybe fatal
-   * error) and return */
-
-  /* switch on type of macro:
-      - '!' syntax macro (inner switch)
-	  - procedural macro - "A token-based function-like macro"
-	  - 'macro_rules' (by example/pattern-match) macro? or not? "an
-     AST-based function-like macro"
-	  - else is unreachable
-      - attribute syntax macro (inner switch)
-	  - procedural macro attribute syntax - "A token-based attribute macro"
-	  - legacy macro attribute syntax? - "an AST-based attribute macro"
-	  - non-macro attribute: mark known
-	  - else is unreachable
-      - derive macro (inner switch)
-	  - derive or legacy derive - "token-based" vs "AST-based"
-	  - else is unreachable
-      - derive container macro - unreachable*/
-
-#if 0
-  // macro_rules macro test code
-  auto rule_def = find_rules_def(invoc->get_path());
-  if (rule_def != nullptr) {
-    ASTFrag expanded = expand_decl_macro(invoc, rule_def);
-    /* could make this a data structure containing vectors of exprs, patterns and types (for regular),
-     * and then stmts and items (for semi). Except what about having an expr, then a type? Hmm. Might
-     * have to do the "unified base type" thing OR just have a simulated union, and then have AST frag
-     * be a vector of these simulated unions. */
-
-    // how would errors be signalled? null fragment? something else?
-    // what about error vs just not having stuff in rules definition yet?
-
-    /* replace macro invocation with ast frag. actually, don't have any context here. maybe attach ast
-     * frag to macro invocation, and then have a method above get it? Or just return the ast frag from
-     * this method. */
-  }
-#endif
+  if (depth_exceeds_recursion_limit ())
+    {
+      rust_error_at (invoc.get_locus (), "reached recursion limit");
+      return;
+    }
+
+  AST::MacroInvocData &invoc_data = invoc.get_invoc_data ();
+
+  // ??
+  // switch on type of macro:
+  //  - '!' syntax macro (inner switch)
+  //      - procedural macro - "A token-based function-like macro"
+  //      - 'macro_rules' (by example/pattern-match) macro? or not? "an
+  // AST-based function-like macro"
+  //      - else is unreachable
+  //  - attribute syntax macro (inner switch)
+  //  - procedural macro attribute syntax - "A token-based attribute
+  // macro"
+  //      - legacy macro attribute syntax? - "an AST-based attribute macro"
+  //      - non-macro attribute: mark known
+  //      - else is unreachable
+  //  - derive macro (inner switch)
+  //      - derive or legacy derive - "token-based" vs "AST-based"
+  //      - else is unreachable
+  //  - derive container macro - unreachable
+
+  // lookup the rules for this macro
+  NodeId resolved_node = UNKNOWN_NODEID;
+  bool found = resolver->get_macro_scope ().lookup (
+    Resolver::CanonicalPath::new_seg (invoc.get_pattern_node_id (),
+				      invoc_data.get_path ().as_string ()),
+    &resolved_node);
+  if (!found)
+    {
+      rust_error_at (invoc.get_locus (), "unknown macro");
+      return;
+    }
+
+  // lookup the rules
+  AST::MacroRulesDefinition *rules_def = nullptr;
+  bool ok = mappings->lookup_macro_def (resolved_node, &rules_def);
+  rust_assert (ok);
+
+  auto fragment
+    = expand_decl_macro (invoc.get_locus (), invoc_data, *rules_def);
+
+  // lets attach this fragment to the invocation
+  invoc.set_fragment (std::move (fragment));
 }
 
 /* Determines whether any cfg predicate is false and hence item with attributes
@@ -3225,6 +3276,9 @@ MacroExpander::expand_cfg_attrs (AST::AttrVec &attrs)
 void
 MacroExpander::expand_crate ()
 {
+  NodeId scope_node_id = crate.get_node_id ();
+  resolver->get_macro_scope ().push (scope_node_id);
+
   /* fill macro/decorator map from init list? not sure where init list comes
    * from? */
 
@@ -3267,4 +3321,396 @@ MacroExpander::expand_crate ()
 
   // extract exported macros?
 }
+
+bool
+MacroExpander::depth_exceeds_recursion_limit () const
+{
+  return expansion_depth >= cfg.recursion_limit;
+}
+
+bool
+MacroExpander::try_match_rule (AST::MacroRule &match_rule,
+			       AST::DelimTokenTree &invoc_token_tree)
+{
+  MacroInvocLexer lex (invoc_token_tree.to_token_stream ());
+  Parser<MacroInvocLexer> parser (std::move (lex));
+
+  AST::MacroMatcher &matcher = match_rule.get_matcher ();
+
+  expansion_depth++;
+  if (!match_matcher (parser, matcher))
+    {
+      expansion_depth--;
+      return false;
+    }
+  expansion_depth--;
+
+  bool used_all_input_tokens = parser.skip_token (END_OF_FILE);
+  return used_all_input_tokens;
+}
+
+bool
+MacroExpander::match_fragment (Parser<MacroInvocLexer> &parser,
+			       AST::MacroMatchFragment &fragment)
+{
+  switch (fragment.get_frag_spec ())
+    {
+    case AST::MacroFragSpec::EXPR:
+      parser.parse_expr ();
+      break;
+
+    case AST::MacroFragSpec::BLOCK:
+      parser.parse_block_expr ();
+      break;
+
+    case AST::MacroFragSpec::IDENT:
+      parser.parse_identifier_pattern ();
+      break;
+
+    case AST::MacroFragSpec::LITERAL:
+      parser.parse_literal_expr ();
+      break;
+
+    case AST::MacroFragSpec::ITEM:
+      parser.parse_item (false);
+      break;
+
+    case AST::MacroFragSpec::TY:
+      parser.parse_type ();
+      break;
+
+    case AST::MacroFragSpec::PAT:
+      parser.parse_pattern ();
+      break;
+
+    case AST::MacroFragSpec::PATH:
+      parser.parse_path_in_expression ();
+      break;
+
+    case AST::MacroFragSpec::VIS:
+      parser.parse_visibility ();
+      break;
+
+    case AST::MacroFragSpec::STMT:
+      parser.parse_stmt ();
+      break;
+
+    case AST::MacroFragSpec::LIFETIME:
+      parser.parse_lifetime_params ();
+      break;
+
+      // is meta attributes?
+    case AST::MacroFragSpec::META:
+      gcc_unreachable ();
+      break;
+
+      // what is TT?
+    case AST::MacroFragSpec::TT:
+      gcc_unreachable ();
+      break;
+
+      // i guess we just ignore invalid and just error out
+    case AST::MacroFragSpec::INVALID:
+      return false;
+    }
+
+  // it matches if the parser did not produce errors trying to parse that type
+  // of item
+  return !parser.has_errors ();
+}
+
+bool
+MacroExpander::match_matcher (Parser<MacroInvocLexer> &parser,
+			      AST::MacroMatcher &matcher)
+{
+  if (depth_exceeds_recursion_limit ())
+    {
+      // FIXME location
+      rust_error_at (Location (), "reached recursion limit");
+      return false;
+    }
+
+  // this is used so we can check that we delimit the stream correctly.
+  switch (matcher.get_delim_type ())
+    {
+      case AST::DelimType::PARENS: {
+	if (!parser.skip_token (LEFT_PAREN))
+	  return false;
+      }
+      break;
+
+      case AST::DelimType::SQUARE: {
+	if (!parser.skip_token (LEFT_SQUARE))
+	  return false;
+      }
+      break;
+
+      case AST::DelimType::CURLY: {
+	if (!parser.skip_token (LEFT_CURLY))
+	  return false;
+      }
+      break;
+    }
+
+  const MacroInvocLexer &source = parser.get_token_source ();
+
+  for (auto &match : matcher.get_matches ())
+    {
+      size_t offs_begin = source.get_offs ();
+      switch (match->get_macro_match_type ())
+	{
+	  case AST::MacroMatch::MacroMatchType::Fragment: {
+	    AST::MacroMatchFragment *fragment
+	      = static_cast<AST::MacroMatchFragment *> (match.get ());
+	    if (!match_fragment (parser, *fragment))
+	      return false;
+
+	    // matched fragment get the offset in the token stream
+	    size_t offs_end = source.get_offs ();
+	    sub_stack.peek ().insert (
+	      {fragment->get_ident (),
+	       {fragment->get_ident (), offs_begin, offs_end}});
+	  }
+	  break;
+
+	  case AST::MacroMatch::MacroMatchType::Tok: {
+	    AST::Token *tok = static_cast<AST::Token *> (match.get ());
+	    if (!match_token (parser, *tok))
+	      return false;
+	  }
+	  break;
+
+	  case AST::MacroMatch::MacroMatchType::Repetition: {
+	    AST::MacroMatchRepetition *rep
+	      = static_cast<AST::MacroMatchRepetition *> (match.get ());
+	    if (!match_repetition (parser, *rep))
+	      return false;
+	  }
+	  break;
+
+	  case AST::MacroMatch::MacroMatchType::Matcher: {
+	    AST::MacroMatcher *m
+	      = static_cast<AST::MacroMatcher *> (match.get ());
+	    expansion_depth++;
+	    if (!match_matcher (parser, *m))
+	      {
+		expansion_depth--;
+		return false;
+	      }
+	    expansion_depth--;
+	  }
+	  break;
+	}
+    }
+
+  switch (matcher.get_delim_type ())
+    {
+      case AST::DelimType::PARENS: {
+	if (!parser.skip_token (RIGHT_PAREN))
+	  return false;
+      }
+      break;
+
+      case AST::DelimType::SQUARE: {
+	if (!parser.skip_token (RIGHT_SQUARE))
+	  return false;
+      }
+      break;
+
+      case AST::DelimType::CURLY: {
+	if (!parser.skip_token (RIGHT_CURLY))
+	  return false;
+      }
+      break;
+    }
+
+  return true;
+}
+
+bool
+MacroExpander::match_token (Parser<MacroInvocLexer> &parser, AST::Token &token)
+{
+  // FIXME this needs to actually match the content and the type
+  return parser.skip_token (token.get_id ());
+}
+
+bool
+MacroExpander::match_repetition (Parser<MacroInvocLexer> &parser,
+				 AST::MacroMatchRepetition &rep)
+{
+  // TODO
+  gcc_unreachable ();
+  return false;
+}
+
+AST::ASTFragment
+MacroExpander::transcribe_rule (
+  AST::MacroRule &match_rule, AST::DelimTokenTree &invoc_token_tree,
+  std::map<std::string, MatchedFragment> &matched_fragments)
+{
+  // we can manipulate the token tree to substitute the dollar identifiers so
+  // that when we call parse its already substituted for us
+  AST::MacroTranscriber &transcriber = match_rule.get_transcriber ();
+  AST::DelimTokenTree &transcribe_tree = transcriber.get_token_tree ();
+
+  auto invoc_stream = invoc_token_tree.to_token_stream ();
+  auto macro_rule_tokens = transcribe_tree.to_token_stream ();
+
+  std::vector<std::unique_ptr<AST::Token>> substituted_tokens
+    = substitute_tokens (invoc_stream, macro_rule_tokens, matched_fragments);
+
+  // handy for debugging
+  // for (auto &tok : substituted_tokens)
+  //   {
+  //     rust_debug ("tok: [%s]", tok->as_string ().c_str ());
+  //   }
+
+  // parse it to an ASTFragment
+  MacroInvocLexer lex (std::move (substituted_tokens));
+  Parser<MacroInvocLexer> parser (std::move (lex));
+
+  // this is used so we can check that we delimit the stream correctly.
+  std::vector<AST::SingleASTNode> nodes;
+  switch (transcribe_tree.get_delim_type ())
+    {
+    case AST::DelimType::PARENS:
+      rust_assert (parser.skip_token (LEFT_PAREN));
+      break;
+
+    case AST::DelimType::CURLY:
+      rust_assert (parser.skip_token (LEFT_CURLY));
+      break;
+
+    case AST::DelimType::SQUARE:
+      rust_assert (parser.skip_token (LEFT_SQUARE));
+      break;
+    }
+
+  // parse the item
+  switch (invoc_token_tree.get_delim_type ())
+    {
+      case AST::DelimType::PARENS: {
+	auto expr = parser.parse_expr ();
+	if (expr != nullptr && !parser.has_errors ())
+	  nodes.push_back (std::move (expr));
+      }
+      break;
+
+      case AST::DelimType::CURLY: {
+	auto item = parser.parse_item (false);
+	if (item != nullptr && !parser.has_errors ())
+	  nodes.push_back (std::move (item));
+      }
+      break;
+
+      case AST::DelimType::SQUARE: {
+	// FIXME
+	gcc_unreachable ();
+      }
+      break;
+    }
+
+  // emit any errors
+  if (parser.has_errors ())
+    {
+      for (auto &err : parser.get_errors ())
+	{
+	  rust_error_at (err.locus, "%s", err.message.c_str ());
+	}
+      return AST::ASTFragment::create_empty ();
+    }
+
+  // are all the tokens used?
+  bool did_delimit = false;
+  switch (transcribe_tree.get_delim_type ())
+    {
+    case AST::DelimType::PARENS:
+      did_delimit = parser.skip_token (RIGHT_PAREN);
+      break;
+    case AST::DelimType::SQUARE:
+      did_delimit = parser.skip_token (RIGHT_SQUARE);
+      break;
+    case AST::DelimType::CURLY:
+      did_delimit = parser.skip_token (RIGHT_CURLY);
+      break;
+    }
+
+  bool reached_end_of_stream = did_delimit && parser.skip_token (END_OF_FILE);
+  if (!reached_end_of_stream)
+    {
+      const_TokenPtr current_token = parser.peek_current_token ();
+      rust_error_at (current_token->get_locus (),
+		     "tokens here and after are unparsed");
+    }
+
+  return AST::ASTFragment (std::move (nodes));
+}
+
+std::vector<std::unique_ptr<AST::Token>>
+MacroExpander::substitute_tokens (
+  std::vector<std::unique_ptr<AST::Token>> &input,
+  std::vector<std::unique_ptr<AST::Token>> &macro,
+  std::map<std::string, MatchedFragment> &fragments)
+{
+  std::vector<std::unique_ptr<AST::Token>> replaced_tokens;
+
+  for (size_t i = 0; i < macro.size (); i++)
+    {
+      auto &tok = macro.at (i);
+      if (tok->get_id () == DOLLAR_SIGN)
+	{
+	  std::vector<std::unique_ptr<AST::Token>> parsed_toks;
+
+	  std::string ident;
+	  for (size_t offs = i; i < macro.size (); offs++)
+	    {
+	      auto &tok = macro.at (offs);
+	      if (tok->get_id () == DOLLAR_SIGN && offs == i)
+		{
+		  parsed_toks.push_back (tok->clone_token ());
+		}
+	      else if (tok->get_id () == IDENTIFIER)
+		{
+		  rust_assert (tok->as_string ().size () == 1);
+		  ident.push_back (tok->as_string ().at (0));
+		  parsed_toks.push_back (tok->clone_token ());
+		}
+	      else
+		{
+		  break;
+		}
+	    }
+
+	  // lookup the ident
+	  auto it = fragments.find (ident);
+	  if (it == fragments.end ())
+	    {
+	      // just leave the tokens in
+	      for (auto &tok : parsed_toks)
+		{
+		  replaced_tokens.push_back (tok->clone_token ());
+		}
+	    }
+	  else
+	    {
+	      // replace
+	      MatchedFragment &frag = it->second;
+	      for (size_t offs = frag.token_offset_begin;
+		   offs < frag.token_offset_end; offs++)
+		{
+		  auto &tok = input.at (offs);
+		  replaced_tokens.push_back (tok->clone_token ());
+		}
+	    }
+	  i += parsed_toks.size () - 1;
+	}
+      else
+	{
+	  replaced_tokens.push_back (tok->clone_token ());
+	}
+    }
+
+  return replaced_tokens;
+}
+
 } // namespace Rust
diff --git a/gcc/rust/expand/rust-macro-expand.h b/gcc/rust/expand/rust-macro-expand.h
index cd9165a..0f13f9e 100644
--- a/gcc/rust/expand/rust-macro-expand.h
+++ b/gcc/rust/expand/rust-macro-expand.h
@@ -19,8 +19,13 @@
 #ifndef RUST_MACRO_EXPAND_H
 #define RUST_MACRO_EXPAND_H
 
+#include "rust-buffered-queue.h"
+#include "rust-parse.h"
+#include "rust-token.h"
 #include "rust-ast.h"
 #include "rust-macro.h"
+#include "rust-hir-map.h"
+#include "rust-name-resolver.h"
 
 // Provides objects and method prototypes for macro expansion
 
@@ -43,6 +48,85 @@ struct ExpansionCfg
   std::string crate_name = "";
 };
 
+class MacroInvocLexer
+{
+public:
+  MacroInvocLexer (std::vector<std::unique_ptr<AST::Token>> stream)
+    : offs (0), token_stream (std::move (stream))
+  {}
+
+  // Returns token n tokens ahead of current position.
+  const_TokenPtr peek_token (int n)
+  {
+    if ((offs + n) >= token_stream.size ())
+      return Token::make (END_OF_FILE, Location ());
+
+    return token_stream.at (offs + n)->get_tok_ptr ();
+  }
+  // Peeks the current token.
+  const_TokenPtr peek_token () { return peek_token (0); }
+
+  // Advances current token to n + 1 tokens ahead of current position.
+  void skip_token (int n) { offs += (n + 1); }
+
+  // Skips the current token.
+  void skip_token () { skip_token (0); }
+
+  // Splits the current token into two. Intended for use with nested generics
+  // closes (i.e. T<U<X>> where >> is wrongly lexed as one token). Note that
+  // this will only work with "simple" tokens like punctuation.
+  void split_current_token (TokenId /*new_left*/, TokenId /*new_right*/)
+  {
+    // FIXME
+    gcc_unreachable ();
+  }
+
+  std::string get_filename () const
+  {
+    gcc_unreachable ();
+    return "FIXME";
+  }
+
+  size_t get_offs () const { return offs; }
+
+private:
+  size_t offs;
+  std::vector<std::unique_ptr<AST::Token>> token_stream;
+};
+
+struct MatchedFragment
+{
+  std::string fragment_ident;
+  size_t token_offset_begin;
+  size_t token_offset_end;
+
+  std::string as_string () const
+  {
+    return fragment_ident + "=" + std::to_string (token_offset_begin) + ":"
+	   + std::to_string (token_offset_end);
+  }
+};
+
+class SubstitutionScope
+{
+public:
+  SubstitutionScope () : stack () {}
+
+  void push () { stack.push_back ({}); }
+
+  std::map<std::string, MatchedFragment> pop ()
+  {
+    auto top = stack.back ();
+    stack.pop_back ();
+    return top;
+  }
+
+  std::map<std::string, MatchedFragment> &peek () { return stack.back (); }
+
+private:
+  std::vector<std::map<std::string, MatchedFragment>> stack;
+};
+
 // Object used to store shared data (between functions) for macro expansion.
 struct MacroExpander
 {
@@ -50,7 +134,9 @@ struct MacroExpander
   unsigned int expansion_depth = 0;
 
   MacroExpander (AST::Crate &crate, ExpansionCfg cfg, Session &session)
-    : cfg (cfg), crate (crate), session (session)
+    : cfg (cfg), crate (crate), session (session),
+      sub_stack (SubstitutionScope ()), resolver (Resolver::Resolver::get ()),
+      mappings (Analysis::Mappings::get ())
   {}
 
   ~MacroExpander () = default;
@@ -61,10 +147,11 @@ struct MacroExpander
   /* Expands a macro invocation (not macro invocation semi) - possibly make both
    * have similar duck-typed interface and use templates?*/
   // should this be public or private?
-  void expand_invoc (std::unique_ptr<AST::MacroInvocation> &invoc);
+  void expand_invoc (AST::MacroInvocation &invoc);
 
   // Expands a single declarative macro.
-  AST::ASTFragment expand_decl_macro (AST::MacroInvocData &invoc,
+  AST::ASTFragment expand_decl_macro (Location locus,
+				      AST::MacroInvocData &invoc,
 				      AST::MacroRulesDefinition &rules_def);
 
   void expand_cfg_attrs (AST::AttrVec &attrs);
@@ -76,10 +163,42 @@ struct MacroExpander
   // Get the literal representation of a cfg! macro.
   AST::Literal expand_cfg_macro (AST::MacroInvocData &invoc);
 
+  bool depth_exceeds_recursion_limit () const;
+
+  bool try_match_rule (AST::MacroRule &match_rule,
+		       AST::DelimTokenTree &invoc_token_tree);
+
+  AST::ASTFragment
+  transcribe_rule (AST::MacroRule &match_rule,
+		   AST::DelimTokenTree &invoc_token_tree,
+		   std::map<std::string, MatchedFragment> &matched_fragments);
+
+  bool match_fragment (Parser<MacroInvocLexer> &parser,
+		       AST::MacroMatchFragment &fragment);
+
+  bool match_token (Parser<MacroInvocLexer> &parser, AST::Token &token);
+
+  bool match_repetition (Parser<MacroInvocLexer> &parser,
+			 AST::MacroMatchRepetition &rep);
+
+  bool match_matcher (Parser<MacroInvocLexer> &parser,
+		      AST::MacroMatcher &matcher);
+
+  static std::vector<std::unique_ptr<AST::Token>>
+  substitute_tokens (std::vector<std::unique_ptr<AST::Token>> &input,
+		     std::vector<std::unique_ptr<AST::Token>> &macro,
+		     std::map<std::string, MatchedFragment> &fragments);
+
 private:
   AST::Crate &crate;
   Session &session;
+  SubstitutionScope sub_stack;
+
+public:
+  Resolver::Resolver *resolver;
+  Analysis::Mappings *mappings;
 };
+
 } // namespace Rust
 
 #endif
diff --git a/gcc/rust/hir/rust-ast-lower-expr.h b/gcc/rust/hir/rust-ast-lower-expr.h
index 1cc3f1d..a8048bb 100644
--- a/gcc/rust/hir/rust-ast-lower-expr.h
+++ b/gcc/rust/hir/rust-ast-lower-expr.h
@@ -100,6 +100,16 @@ public:
     return resolver.translated;
   }
 
+  void visit (AST::MacroInvocation &expr) override
+  {
+    AST::ASTFragment &fragment = expr.get_fragment ();
+
+    // FIXME
+    // this assertion might go away, maybe on failure's to expand a macro?
+    rust_assert (!fragment.get_nodes ().empty ());
+    fragment.get_nodes ().at (0).accept_vis (*this);
+  }
+
   void visit (AST::TupleIndexExpr &expr) override
   {
     HIR::Expr *tuple_expr
diff --git a/gcc/rust/parse/rust-parse.cc b/gcc/rust/parse/rust-parse.cc
index f2c1301..f995e4b 100644
--- a/gcc/rust/parse/rust-parse.cc
+++ b/gcc/rust/parse/rust-parse.cc
@@ -18,32 +18,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "rust-linemap.h"
 #include "rust-diagnostics.h"
 
-#if 0
-#include "config.h"
-#include "system.h"
-#include "coretypes.h"
-#include "target.h"
-#include "tree.h"
-#include "tree-iterator.h"
-#include "input.h"
-#include "diagnostic.h"
-#include "stringpool.h"
-#include "cgraph.h"
-#include "gimplify.h"
-#include "gimple-expr.h"
-#include "convert.h"
-#include "print-tree.h"
-#include "stor-layout.h"
-#include "fold-const.h"
-/* order: config, system, coretypes, target, tree, tree-iterator, input, diagnostic, stringpool,
- * cgraph, gimplify, gimple-expr, convert, print-tree, stor-layout, fold-const  */
-// probably don't need all these
-#endif
-// maybe put these back in if compiling no longer works
-
-/* TODO: move non-essential stuff back here from rust-parse-impl.h after
- * confirming that it works */
-
 namespace Rust {
 
 std::string
diff --git a/gcc/rust/parse/rust-parse.h b/gcc/rust/parse/rust-parse.h
index acab7ff..5ee7b4e 100644
--- a/gcc/rust/parse/rust-parse.h
+++ b/gcc/rust/parse/rust-parse.h
@@ -86,6 +86,29 @@ struct ParseRestrictions
 // TODO: if updated to C++20, ManagedTokenSource would be useful as a concept
 template <typename ManagedTokenSource> class Parser
 {
+public:
+  bool skip_token (TokenId t);
+
+  std::unique_ptr<AST::Expr>
+  parse_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
+	      ParseRestrictions restrictions = ParseRestrictions ());
+
+  std::unique_ptr<AST::LiteralExpr> parse_literal_expr (AST::AttrVec outer_attrs
+							= AST::AttrVec ());
+
+  std::unique_ptr<AST::BlockExpr>
+  parse_block_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
+		    Location pratt_parsed_loc = Linemap::unknown_location ());
+
+  std::unique_ptr<AST::Item> parse_item (bool called_from_statement);
+  std::unique_ptr<AST::Pattern> parse_pattern ();
+  std::unique_ptr<AST::Stmt> parse_stmt ();
+  std::unique_ptr<AST::Type> parse_type ();
+  AST::PathInExpression parse_path_in_expression ();
+  std::vector<std::unique_ptr<AST::LifetimeParam> > parse_lifetime_params ();
+  AST::Visibility parse_visibility ();
+  std::unique_ptr<AST::IdentifierPattern> parse_identifier_pattern ();
+
 private:
   void skip_after_semicolon ();
   void skip_after_end ();
@@ -93,7 +116,6 @@ private:
   void skip_after_next_block ();
   void skip_after_end_attribute ();
 
-  bool skip_token (TokenId t);
   const_TokenPtr expect_token (TokenId t);
   void unexpected_token (const_TokenPtr t);
   bool skip_generics_right_angle ();
@@ -118,7 +140,6 @@ private:
   AST::GenericArgs parse_path_generic_args ();
   AST::GenericArgsBinding parse_generic_args_binding ();
   AST::TypePathFunction parse_type_path_function ();
-  AST::PathInExpression parse_path_in_expression ();
   AST::PathExprSegment parse_path_expr_segment ();
   AST::QualifiedPathInExpression
   // When given a pratt_parsed_loc, use it as the location of the
@@ -147,10 +168,8 @@ private:
   std::unique_ptr<AST::MacroMatchRepetition> parse_macro_match_repetition ();
 
   // Top-level item-related
-  std::unique_ptr<AST::Item> parse_item (bool called_from_statement);
   std::unique_ptr<AST::VisItem> parse_vis_item (AST::AttrVec outer_attrs);
   std::unique_ptr<AST::MacroItem> parse_macro_item (AST::AttrVec outer_attrs);
-  AST::Visibility parse_visibility ();
 
   // VisItem subclass-related
   std::unique_ptr<AST::Module> parse_module (AST::Visibility vis,
@@ -169,7 +188,7 @@ private:
   template <typename EndTokenPred>
   std::vector<std::unique_ptr<AST::GenericParam> >
   parse_generic_params (EndTokenPred is_end_token);
-  std::vector<std::unique_ptr<AST::LifetimeParam> > parse_lifetime_params ();
+
   template <typename EndTokenPred>
   std::vector<std::unique_ptr<AST::LifetimeParam> >
   parse_lifetime_params (EndTokenPred is_end_token);
@@ -260,9 +279,6 @@ private:
 
   // Expression-related (Pratt parsed)
   std::unique_ptr<AST::Expr>
-  parse_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
-	      ParseRestrictions restrictions = ParseRestrictions ());
-  std::unique_ptr<AST::Expr>
   parse_expr (int right_binding_power,
 	      AST::AttrVec outer_attrs = AST::AttrVec (),
 	      ParseRestrictions restrictions = ParseRestrictions ());
@@ -478,9 +494,6 @@ private:
   // When given a pratt_parsed_loc, use it as the location of the
   // first token parsed in the expression (the parsing of that first
   // token should be skipped).
-  std::unique_ptr<AST::BlockExpr>
-  parse_block_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
-		    Location pratt_parsed_loc = Linemap::unknown_location ());
   std::unique_ptr<AST::IfExpr>
   parse_if_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
 		 Location pratt_parsed_loc = Linemap::unknown_location ());
@@ -518,8 +531,7 @@ private:
   std::unique_ptr<AST::ClosureExpr> parse_closure_expr (AST::AttrVec outer_attrs
 							= AST::AttrVec ());
   AST::ClosureParam parse_closure_param ();
-  std::unique_ptr<AST::LiteralExpr> parse_literal_expr (AST::AttrVec outer_attrs
-							= AST::AttrVec ());
+
   // When given a pratt_parsed_loc, use it as the location of the
   // first token parsed in the expression (the parsing of that first
   // token should be skipped).
@@ -548,7 +560,6 @@ private:
   bool will_be_expr_with_block ();
 
   // Type-related
-  std::unique_ptr<AST::Type> parse_type ();
   std::unique_ptr<AST::TypeNoBounds> parse_type_no_bounds ();
   std::unique_ptr<AST::TypeNoBounds> parse_slice_or_array_type ();
   std::unique_ptr<AST::RawPointerType> parse_raw_pointer_type ();
@@ -561,7 +572,6 @@ private:
   AST::MaybeNamedParam parse_maybe_named_param (AST::AttrVec outer_attrs);
 
   // Statement-related
-  std::unique_ptr<AST::Stmt> parse_stmt ();
   std::unique_ptr<AST::LetStmt> parse_let_stmt (AST::AttrVec outer_attrs);
   std::unique_ptr<AST::ExprStmt> parse_expr_stmt (AST::AttrVec outer_attrs);
   std::unique_ptr<AST::ExprStmtWithBlock>
@@ -574,13 +584,11 @@ private:
   ExprOrStmt parse_path_based_stmt_or_expr (AST::AttrVec outer_attrs);
 
   // Pattern-related
-  std::unique_ptr<AST::Pattern> parse_pattern ();
   std::unique_ptr<AST::Pattern> parse_literal_or_range_pattern ();
   std::unique_ptr<AST::RangePatternBound> parse_range_pattern_bound ();
   std::unique_ptr<AST::ReferencePattern> parse_reference_pattern ();
   std::unique_ptr<AST::Pattern> parse_grouped_or_tuple_pattern ();
   std::unique_ptr<AST::SlicePattern> parse_slice_pattern ();
-  std::unique_ptr<AST::IdentifierPattern> parse_identifier_pattern ();
   std::unique_ptr<AST::Pattern> parse_ident_leading_pattern ();
   std::unique_ptr<AST::TupleStructItems> parse_tuple_struct_items ();
   AST::StructPatternElements parse_struct_pattern_elems ();
@@ -617,6 +625,10 @@ public:
   // Get a reference to the list of errors encountered
   std::vector<Error> &get_errors () { return error_table; }
 
+  const ManagedTokenSource &get_token_source () const { return lexer; }
+
+  const_TokenPtr peek_current_token () { return lexer.peek_token (0); }
+
 private:
   // The token source (usually lexer) associated with the parser.
   ManagedTokenSource lexer;
diff --git a/gcc/rust/resolve/rust-ast-resolve-expr.h b/gcc/rust/resolve/rust-ast-resolve-expr.h
index b7b8646..4ccb72b 100644
--- a/gcc/rust/resolve/rust-ast-resolve-expr.h
+++ b/gcc/rust/resolve/rust-ast-resolve-expr.h
@@ -70,6 +70,16 @@ public:
     expr->accept_vis (resolver);
   };
 
+  void visit (AST::MacroInvocation &expr) override
+  {
+    AST::ASTFragment &fragment = expr.get_fragment ();
+
+    // FIXME
+    // this assertion might go away, maybe on failure's to expand a macro?
+    rust_assert (!fragment.get_nodes ().empty ());
+    fragment.get_nodes ().at (0).accept_vis (*this);
+  }
+
   void visit (AST::TupleIndexExpr &expr) override
   {
     resolve_expr (expr.get_tuple_expr ().get (), expr.get_node_id ());
diff --git a/gcc/testsuite/rust/execute/torture/macros1.rs b/gcc/testsuite/rust/execute/torture/macros1.rs
new file mode 100644
index 0000000..652d2d8
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/macros1.rs
@@ -0,0 +1,13 @@
+macro_rules! add {
+    ($a:expr,$b:expr) => {
+        $a + $b
+    };
+}
+
+fn test() -> i32 {
+    add!(1 + 2, 3)
+}
+
+fn main() -> i32 {
+    test() - 6
+}
diff --git a/gcc/testsuite/rust/execute/torture/macros2.rs b/gcc/testsuite/rust/execute/torture/macros2.rs
new file mode 100644
index 0000000..0116bd1
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/macros2.rs
@@ -0,0 +1,40 @@
+// { dg-output "arg\narg\narg\n" }
+extern "C" {
+    fn printf(s: *const i8, ...);
+}
+
+fn f() {
+    unsafe {
+        let r_s = "arg\n\0";
+        let s_p = r_s as *const str;
+        let c_p = s_p as *const i8;
+
+        printf(c_p);
+    }
+}
+
+macro_rules! kw0 {
+    (keyword) => {
+        f()
+    };
+}
+
+macro_rules! kw1 {
+    (fn) => {
+        f()
+    };
+}
+
+macro_rules! kw2 {
+    (kw0 kw1 kw3) => {
+        f()
+    };
+}
+
+fn main() -> i32 {
+    kw0!(keyword);
+    kw1!(fn);
+    kw2!(kw0 kw1 kw3);
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/macros3.rs b/gcc/testsuite/rust/execute/torture/macros3.rs
new file mode 100644
index 0000000..c1f5a53
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/macros3.rs
@@ -0,0 +1,61 @@
+// { dg-output "invok\ninvok\ninvok\ninvok\ninvok\n" }
+extern "C" {
+    fn printf(s: *const i8, ...);
+}
+
+fn f() {
+    unsafe {
+        let r_s = "invok\n\0";
+        let s_p = r_s as *const str;
+        let c_p = s_p as *const i8;
+
+        printf(c_p);
+    }
+}
+
+macro_rules! invocation0 {
+    (valid) => {
+        f()
+    };
+    () => {};
+}
+
+macro_rules! invocation1 {
+    (valid) => {};
+    () => {
+        f()
+    };
+}
+
+macro_rules! invocation2 {
+    (valid) => {
+        f()
+    };
+    (invalid) => {};
+}
+
+macro_rules! invocation3 {
+    (this is a valid invocation) => {
+        f()
+    };
+    (not this one) => {};
+}
+
+macro_rules! invocation4 {
+    (fn f() {}) => {
+        f()
+    };
+    (not a keyword) => {};
+}
+
+fn main() -> i32 {
+    invocation0!(valid);
+    invocation1!();
+    invocation2!(valid);
+    invocation3!(this is a valid invocation);
+    invocation4!(
+        fn f() {}
+    );
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/macros4.rs b/gcc/testsuite/rust/execute/torture/macros4.rs
new file mode 100644
index 0000000..3303bfa
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/macros4.rs
@@ -0,0 +1,15 @@
+macro_rules! add {
+    ($a:expr,$b:expr) => {
+        $a + $b
+    };
+    ($a:expr) => {
+        $a
+    };
+}
+
+fn main() -> i32 {
+    let mut x = add!(1);
+    x += add!(2, 3);
+
+    x - 6
+}
diff --git a/gcc/testsuite/rust/execute/xfail/macro2.rs b/gcc/testsuite/rust/execute/xfail/macro2.rs
deleted file mode 100644
index 49bd6a8..0000000
--- a/gcc/testsuite/rust/execute/xfail/macro2.rs
+++ /dev/null
@@ -1,30 +0,0 @@
-// { dg-output "arg\narg\n" }
-extern "C" {
-    fn printf(s: *const i8, ...);
-}
-
-fn f() {
-    let r_s = "arg\n\0";
-    let s_p = r_s as *const str;
-    let c_p = s_p as *const i8;
-
-    printf(c_p);
-}
-
-macro_rules! kw0 {
-    (keyword) => { f() };
-}
-
-macro_rules! kw1 {
-    (fn) => { f() };
-}
-
-macro_rules! kw2 {
-    (kw0 kw1 kw3) => { f() };
-}
-
-fn main() {
-    kw0!(keyword);
-    kw1!(fn);
-    kw2!(kw0 kw1 kw3);
-}
diff --git a/gcc/testsuite/rust/execute/xfail/macro3.rs b/gcc/testsuite/rust/execute/xfail/macro3.rs
deleted file mode 100644
index 0d99d716..0000000
--- a/gcc/testsuite/rust/execute/xfail/macro3.rs
+++ /dev/null
@@ -1,45 +0,0 @@
-// { dg-output "invok\ninvok\ninvok\ninvok\ninvok\n" }
-extern "C" {
-    fn printf(s: *const i8, ...);
-}
-
-fn f() {
-    let r_s = "invok\n\0";
-    let s_p = r_s as *const str;
-    let c_p = s_p as *const i8;
-
-    printf(c_p);
-}
-
-macro_rules! invocation0 {
-    (valid) => { f() };
-    () => { };
-}
-
-macro_rules! invocation1 {
-    (valid) => { };
-    () => { f() };
-}
-
-macro_rules! invocation2 {
-    (valid) => { f() };
-    (invalid) => { };
-}
-
-macro_rules! invocation3 {
-    (this is a valid invocation) => { f() };
-    (not this one) => { };
-}
-
-macro_rules! invocation4 {
-    (fn f() {}) => { f() };
-    (not a keyword) => { };
-}
-
-fn main() {
-    invocation0!(valid);
-    invocation1!();
-    invocation2!(valid);
-    invocation3!(this is a valid invocation);
-    invocation4!(fn f() {});
-}