expand: Add derive proc macro draft

Add a first draft for derive proc macros based on the attribute expander
code. Convert the result back to a parsable entity and parse it.
The procedural macro expander was beginning to require almost all
functionalities already provided by the macro expander, hence the merge.

gcc/rust/ChangeLog:

	* ast/rust-ast-collector.h: Update enum name to match
	definition.
	* expand/rust-expand-visitor.cc (ExpandVisitor::expand_derive):
	Add call to expander.
	(ExpandVisitor::expand_outer_attribute):
	Change call to macro expander.
	(ExpandVisitor::expand_inner_attribute): Likewise.
	* expand/rust-expand-visitor.h: Remove const attribute to match
	visitor declaration. Attach result to the AST. Add condition for
	item erasure.
	* expand/rust-proc-macro.h: Add token collector and expansion
	call. Add lexers and parsers for each proc macro type and uses
	them to parse macro output.
	* expand/rust-macro-expand.h (struct MacroExpander): Add
	functions.
	* expand/rust-proc-macro.cc (ProcMacroExpander::import_proc_macros):
	Moved from here...
	* expand/rust-macro-expand.cc (MacroExpander::import_proc_macros):
	... to here. Unify procedural macro parsing under one function.
	Add a flag to determine whether it originate from a derive
	macro.
	(MacroExpander::parse_procmacro_output):
	Parse macro output to statements. Store an error on parsing
	failure.
	* Make-lang.in: Add const_TokenPtr specific lexer.
	* expand/rust-proc-macro-invoc-lexer.cc: New file.
	* expand/rust-proc-macro-invoc-lexer.h: New file.
	* rust-session-manager.cc (Session::expansion): Remove
	ProcMacroExpander declaration.
	* ast/rust-ast-fragment.cc (Fragment::Fragment): Add overwrite
	flag.
	(Fragment::should_overwrite): Add a getter to determine whether
	the fragment shall overwrite it's parent or be appended after.
	* ast/rust-ast-fragment.h: Add flag to declaration.

Signed-off-by: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>

13 files changed, 395 insertions, 146 deletions

diff --git a/gcc/rust/Make-lang.in b/gcc/rust/Make-lang.in
index a6245d4..b4342dd 100644
--- a/gcc/rust/Make-lang.in
+++ b/gcc/rust/Make-lang.in
@@ -93,6 +93,7 @@ GRS_OBJS = \
 	rust/rust-derive-copy.o \
 	rust/rust-proc-macro.o \
     rust/rust-macro-invoc-lexer.o \
+    rust/rust-proc-macro-invoc-lexer.o \
     rust/rust-macro-substitute-ctx.o \
     rust/rust-macro-builtins.o \
     rust/rust-hir.o \
diff --git a/gcc/rust/ast/rust-ast-collector.h b/gcc/rust/ast/rust-ast-collector.h
index 91e49ca..feb6c47 100644
--- a/gcc/rust/ast/rust-ast-collector.h
+++ b/gcc/rust/ast/rust-ast-collector.h
@@ -231,7 +231,7 @@ public:
   void visit (EnumItemTuple &item);
   void visit (EnumItemStruct &item);
   void visit (EnumItemDiscriminant &item);
-  void visit (Enum &enum_item);
+  void visit (Enum &enumeration);
   void visit (Union &union_item);
   void visit (ConstantItem &const_item);
   void visit (StaticItem &static_item);
diff --git a/gcc/rust/ast/rust-ast-fragment.cc b/gcc/rust/ast/rust-ast-fragment.cc
index a9dc474..dd2e8a8 100644
--- a/gcc/rust/ast/rust-ast-fragment.cc
+++ b/gcc/rust/ast/rust-ast-fragment.cc
@@ -23,7 +23,8 @@ namespace AST {
 
 Fragment::Fragment (FragmentKind kind, std::vector<SingleASTNode> nodes,
 		    std::vector<std::unique_ptr<AST::Token>> tokens)
-  : kind (kind), nodes (std::move (nodes)), tokens (std::move (tokens))
+  : kind (kind), nodes (std::move (nodes)), tokens (std::move (tokens)),
+    overwrite (true)
 {}
 
 Fragment::Fragment (Fragment const &other) : kind (other.get_kind ())
@@ -46,6 +47,8 @@ Fragment::operator= (Fragment const &other)
   for (auto &t : other.tokens)
     tokens.emplace_back (t->clone_token ());
 
+  overwrite = other.overwrite;
+
   return *this;
 }
 
@@ -56,14 +59,15 @@ Fragment::create_error ()
 }
 
 Fragment::Fragment (std::vector<AST::SingleASTNode> nodes,
-		    std::vector<std::unique_ptr<AST::Token>> tokens)
+		    std::vector<std::unique_ptr<AST::Token>> tokens,
+		    bool overwrite)
   : kind (FragmentKind::Complete), nodes (std::move (nodes)),
-    tokens (std::move (tokens))
+    tokens (std::move (tokens)), overwrite (overwrite)
 {}
 
 Fragment::Fragment (std::vector<AST::SingleASTNode> nodes,
 		    std::unique_ptr<AST::Token> token)
-  : kind (FragmentKind::Complete), nodes (std::move (nodes))
+  : kind (FragmentKind::Complete), nodes (std::move (nodes)), overwrite (true)
 {
   tokens.emplace_back (std::move (token));
 }
@@ -99,6 +103,12 @@ Fragment::should_expand () const
 }
 
 bool
+Fragment::should_overwrite () const
+{
+  return overwrite;
+}
+
+bool
 Fragment::is_expression_fragment () const
 {
   return is_single_fragment_of_kind (SingleASTNode::NodeType::EXPRESSION);
diff --git a/gcc/rust/ast/rust-ast-fragment.h b/gcc/rust/ast/rust-ast-fragment.h
index 41f5a28..36f94db 100644
--- a/gcc/rust/ast/rust-ast-fragment.h
+++ b/gcc/rust/ast/rust-ast-fragment.h
@@ -64,7 +64,8 @@ public:
    * Create a complete AST fragment
    */
   Fragment (std::vector<AST::SingleASTNode> nodes,
-	    std::vector<std::unique_ptr<AST::Token>> tokens);
+	    std::vector<std::unique_ptr<AST::Token>> tokens,
+	    bool overwrite = true);
 
   /**
    * Create a complete AST fragment made of a single token
@@ -78,6 +79,7 @@ public:
 
   bool is_error () const;
   bool should_expand () const;
+  bool should_overwrite () const;
 
   bool is_expression_fragment () const;
   bool is_type_fragment () const;
@@ -110,6 +112,12 @@ private:
   std::vector<std::unique_ptr<AST::Token>> tokens;
 
   /**
+   * Whether the fragment should overwrite the original content. In most case
+   * it should overwrite it, but not with derive procedural macros.
+   */
+  bool overwrite;
+
+  /**
    * We need to make a special case for Expression and Type fragments as only
    * one Node will be extracted from the `nodes` vector
    */
diff --git a/gcc/rust/expand/rust-expand-visitor.cc b/gcc/rust/expand/rust-expand-visitor.cc
index 04a899f..cb6c299 100644
--- a/gcc/rust/expand/rust-expand-visitor.cc
+++ b/gcc/rust/expand/rust-expand-visitor.cc
@@ -1553,7 +1553,7 @@ void
 ExpandVisitor::expand_outer_attribute (T &item, AST::SimplePath &path)
 {
   // FIXME: Retrieve path from segments + local use statements instead of string
-  proc_expander.expand_attribute_proc_macro (item, path);
+  expander.expand_attribute_proc_macro (item, path);
 }
 
 template <typename T>
@@ -1588,7 +1588,7 @@ void
 ExpandVisitor::expand_inner_attribute (T &item, AST::SimplePath &path)
 {
   // FIXME: Retrieve path from segments + local use statements instead of string
-  proc_expander.expand_attribute_proc_macro (item, path);
+  expander.expand_attribute_proc_macro (item, path);
 }
 
 template <typename T>
@@ -1621,15 +1621,15 @@ ExpandVisitor::visit_inner_attrs (T &item)
 
 template <typename T>
 void
-ExpandVisitor::expand_derive (const T &item,
-			      std::unique_ptr<AST::TokenTree> &trait)
+ExpandVisitor::expand_derive (T &item, std::unique_ptr<AST::TokenTree> trait)
 {
-  // FIXME: Implement expansion for that particular trait
+  auto trait_name = trait->as_string ();
+  expander.expand_derive_proc_macro (item, trait_name);
 }
 
 template <typename T>
 void
-ExpandVisitor::expand_derive (const T &item, AST::DelimTokenTree &attr)
+ExpandVisitor::expand_derive (T &item, AST::DelimTokenTree &attr)
 {
   // Item is const because even though the tokenstream might be modified, it
   // should appear as the same input for every derive proc macro.
@@ -1640,7 +1640,7 @@ ExpandVisitor::expand_derive (const T &item, AST::DelimTokenTree &attr)
       for (auto it = trees.begin () + 1; it < trees.end () - 1;
 	   it += 2 /* Increment + skip comma */)
 	{
-	  expand_derive (item, *it);
+	  expand_derive (item, std::move (*it));
 	}
     }
 }
diff --git a/gcc/rust/expand/rust-expand-visitor.h b/gcc/rust/expand/rust-expand-visitor.h
index fd72d0b..0dd5c9a 100644
--- a/gcc/rust/expand/rust-expand-visitor.h
+++ b/gcc/rust/expand/rust-expand-visitor.h
@@ -40,9 +40,7 @@ is_builtin (AST::Attribute &attr);
 class ExpandVisitor : public AST::ASTVisitor
 {
 public:
-  ExpandVisitor (MacroExpander &expander, ProcMacroExpander &proc_expander)
-    : expander (expander), proc_expander (proc_expander)
-  {}
+  ExpandVisitor (MacroExpander &expander) : expander (expander) {}
 
   /* Expand all of the macro invocations currently contained in a crate */
   void go (AST::Crate &crate);
@@ -129,6 +127,8 @@ public:
 	value->accept_vis (*this);
 
 	auto final_fragment = expander.take_expanded_fragment ();
+	auto proc_macro_fragment
+	  = expander.take_expanded_proc_macro_fragment ();
 
 	// FIXME: Is that correct? It seems *extremely* dodgy
 	if (final_fragment.should_expand ())
@@ -144,6 +144,22 @@ public:
 		  }
 	      }
 	  }
+	else if (proc_macro_fragment.should_expand ())
+	  {
+	    if (proc_macro_fragment.should_overwrite ())
+	      it = values.erase (it);
+	    else
+	      it++;
+	    for (auto &node : proc_macro_fragment.get_nodes ())
+	      {
+		auto new_node = extractor (node);
+		if (new_node != nullptr)
+		  {
+		    it = values.insert (it, std::move (new_node));
+		    it++;
+		  }
+	      }
+	  }
 	else
 	  {
 	    ++it;
@@ -368,16 +384,14 @@ public:
   template <typename T> void visit_inner_attrs (T &item);
 
   template <typename T>
-  void expand_derive (const T &item, std::unique_ptr<AST::TokenTree> &trait);
+  void expand_derive (T &item, std::unique_ptr<AST::TokenTree> trait);
 
-  template <typename T>
-  void expand_derive (const T &item, AST::DelimTokenTree &attr);
+  template <typename T> void expand_derive (T &item, AST::DelimTokenTree &attr);
 
   template <typename T> void visit_attrs_with_derive (T &item);
 
 private:
   MacroExpander &expander;
-  ProcMacroExpander &proc_expander;
 };
 
 } // namespace Rust
diff --git a/gcc/rust/expand/rust-macro-expand.cc b/gcc/rust/expand/rust-macro-expand.cc
index c29b9c6..14c6cbc 100644
--- a/gcc/rust/expand/rust-macro-expand.cc
+++ b/gcc/rust/expand/rust-macro-expand.cc
@@ -24,6 +24,8 @@
 #include "rust-parse.h"
 #include "rust-cfg-strip.h"
 #include "rust-early-name-resolver.h"
+#include "rust-session-manager.h"
+#include "rust-proc-macro.h"
 
 namespace Rust {
 
@@ -1049,4 +1051,92 @@ MacroExpander::transcribe_rule (
 
   return fragment;
 }
+
+// TODO: Move to early name resolver ?
+void
+MacroExpander::import_proc_macros (std::string extern_crate)
+{
+  auto path = session.extern_crates.find (extern_crate);
+  if (path == session.extern_crates.end ())
+    {
+      // Extern crate path is not available.
+      // FIXME: Emit error
+      rust_error_at (Location (), "Cannot find requested proc macro crate");
+      gcc_unreachable ();
+    }
+  auto macros = load_macros (path->second);
+
+  std::string prefix = extern_crate + "::";
+  for (auto &macro : macros)
+    {
+      switch (macro.tag)
+	{
+	case ProcMacro::CUSTOM_DERIVE:
+	  rust_debug ("Found one derive proc macro.");
+	  mappings->insert_derive_proc_macro (
+	    std::make_pair (extern_crate,
+			    macro.payload.custom_derive.trait_name),
+	    macro.payload.custom_derive);
+	  break;
+	case ProcMacro::ATTR:
+	  rust_debug ("Found one attribute proc macro.");
+	  mappings->insert_attribute_proc_macro (
+	    std::make_pair (extern_crate, macro.payload.attribute.name),
+	    macro.payload.attribute);
+	  break;
+	case ProcMacro::BANG:
+	  rust_debug ("Found one bang proc macro.");
+	  mappings->insert_bang_proc_macro (
+	    std::make_pair (extern_crate, macro.payload.bang.name),
+	    macro.payload.bang);
+	  break;
+	default:
+	  gcc_unreachable ();
+	}
+    }
+}
+
+void
+MacroExpander::parse_procmacro_output (ProcMacro::TokenStream ts, bool derive)
+{
+  ProcMacroInvocLexer lex (convert (ts));
+  Parser<ProcMacroInvocLexer> parser (lex);
+
+  std::vector<AST::SingleASTNode> nodes;
+  switch (peek_context ())
+    {
+    case ContextType::ITEM:
+      while (lex.peek_token ()->get_id () != END_OF_FILE)
+	{
+	  auto result = parser.parse_item (false);
+	  if (result == nullptr)
+	    break;
+	  nodes.push_back ({std::move (result)});
+	}
+      break;
+    case ContextType::BLOCK:
+      while (lex.peek_token ()->get_id () != END_OF_FILE)
+	{
+	  auto result = parser.parse_stmt ();
+	  if (result == nullptr)
+	    break;
+	  nodes.push_back ({std::move (result)});
+	}
+      break;
+    case ContextType::TRAIT:
+    case ContextType::IMPL:
+    case ContextType::TRAIT_IMPL:
+    case ContextType::EXTERN:
+    case ContextType::TYPE:
+    default:
+      gcc_unreachable ();
+    }
+
+  if (parser.has_errors ())
+    set_expanded_proc_macro_fragment (AST::Fragment::create_error ());
+  else
+    set_expanded_proc_macro_fragment (
+      {nodes, std::vector<std::unique_ptr<AST::Token>> (), !derive});
+}
+
 } // namespace Rust
diff --git a/gcc/rust/expand/rust-macro-expand.h b/gcc/rust/expand/rust-macro-expand.h
index 50277fc..811417a 100644
--- a/gcc/rust/expand/rust-macro-expand.h
+++ b/gcc/rust/expand/rust-macro-expand.h
@@ -28,6 +28,11 @@
 #include "rust-early-name-resolver.h"
 #include "rust-name-resolver.h"
 #include "rust-macro-invoc-lexer.h"
+#include "rust-proc-macro-invoc-lexer.h"
+#include "rust-token-converter.h"
+#include "rust-ast-collector.h"
+#include "rust-system.h"
+#include "libproc_macro/proc_macro.h"
 
 // Provides objects and method prototypes for macro expansion
 
@@ -231,6 +236,7 @@ struct MacroExpander
     : cfg (cfg), crate (crate), session (session),
       sub_stack (SubstitutionScope ()),
       expanded_fragment (AST::Fragment::create_error ()),
+      expanded_proc_macro_fragment (AST::Fragment::create_error ()),
       has_changed_flag (false), resolver (Resolver::Resolver::get ()),
       mappings (Analysis::Mappings::get ())
   {}
@@ -332,6 +338,126 @@ struct MacroExpander
     return fragment;
   }
 
+  void set_expanded_proc_macro_fragment (AST::Fragment &&fragment)
+  {
+    if (!fragment.is_error ())
+      has_changed_flag = true;
+
+    expanded_proc_macro_fragment = std::move (fragment);
+  }
+
+  AST::Fragment take_expanded_proc_macro_fragment ()
+  {
+    auto fragment = std::move (expanded_proc_macro_fragment);
+    expanded_proc_macro_fragment = AST::Fragment::create_error ();
+
+    return fragment;
+  }
+
+  void import_proc_macros (std::string extern_crate);
+
+  template <typename T>
+  void expand_derive_proc_macro (T &item, std::string &trait_name)
+  {
+    ProcMacro::CustomDerive macro;
+
+    // FIXME: Resolve crate name
+    std::string crate = "";
+    std::string name = trait_name;
+
+    if (!mappings->lookup_derive_proc_macro (std::make_pair (crate, name),
+					     macro))
+      {
+	// FIXME: Resolve this path segment instead of taking it directly.
+	import_proc_macros (crate);
+	if (!mappings->lookup_derive_proc_macro (std::make_pair (crate, name),
+						 macro))
+	  {
+	    rust_error_at (Location (), "procedural macro %s not found",
+			   name.c_str ());
+	    rust_assert (false);
+	  }
+      }
+
+    std::vector<TokenPtr> tokens;
+    AST::TokenCollector collector (tokens);
+
+    collector.visit (item);
+
+    auto c = collector.collect_tokens ();
+    std::vector<const_TokenPtr> vec (c.cbegin (), c.cend ());
+
+    parse_procmacro_output (macro.macro (convert (vec)), true);
+  }
+
+  template <typename T>
+  void expand_bang_proc_macro (T &item, AST::SimplePath &path)
+  {
+    ProcMacro::Bang macro;
+
+    std::string crate = path.get_segments ()[0].get_segment_name ();
+    std::string name = path.get_segments ()[1].get_segment_name ();
+    if (!mappings->lookup_bang_proc_macro (std::make_pair (crate, name), macro))
+      {
+	// FIXME: Resolve this path segment instead of taking it directly.
+	import_proc_macros (crate);
+
+	if (!mappings->lookup_bang_proc_macro (std::make_pair (crate, name),
+					       macro))
+	  {
+	    rust_error_at (Location (), "procedural macro %s not found",
+			   name.c_str ());
+	    rust_assert (false);
+	  }
+      }
+
+    std::vector<TokenPtr> tokens;
+    AST::TokenCollector collector (tokens);
+
+    collector.visit (item);
+
+    auto c = collector.collect_tokens ();
+    std::vector<const_TokenPtr> vec (c.cbegin (), c.cend ());
+
+    parse_procmacro_output (macro.macro (convert (vec)), false);
+  }
+
+  template <typename T>
+  void expand_attribute_proc_macro (T &item, AST::SimplePath &path)
+  {
+    ProcMacro::Attribute macro;
+
+    std::string crate = path.get_segments ()[0].get_segment_name ();
+    std::string name = path.get_segments ()[1].get_segment_name ();
+    if (!mappings->lookup_attribute_proc_macro (std::make_pair (crate, name),
+						macro))
+      {
+	// FIXME: Resolve this path segment instead of taking it directly.
+	import_proc_macros (crate);
+	if (!mappings->lookup_attribute_proc_macro (std::make_pair (crate,
+								    name),
+						    macro))
+	  {
+	    rust_error_at (Location (), "procedural macro %s not found",
+			   name.c_str ());
+	    rust_assert (false);
+	  }
+      }
+
+    std::vector<TokenPtr> tokens;
+    AST::TokenCollector collector (tokens);
+
+    collector.visit (item);
+
+    auto c = collector.collect_tokens ();
+    std::vector<const_TokenPtr> vec (c.cbegin (), c.cend ());
+
+    // FIXME: Handle attributes
+    parse_procmacro_output (
+      macro.macro (ProcMacro::TokenStream::make_tokenstream (), convert (vec)),
+      false);
+  }
+
   /**
    * Has the MacroExpander expanded a macro since its state was last reset?
    */
@@ -347,11 +473,14 @@ struct MacroExpander
   AST::MacroInvocation *get_last_invocation () { return last_invoc; }
 
 private:
+  void parse_procmacro_output (ProcMacro::TokenStream ts, bool derive);
+
   AST::Crate &crate;
   Session &session;
   SubstitutionScope sub_stack;
   std::vector<ContextType> context;
   AST::Fragment expanded_fragment;
+  AST::Fragment expanded_proc_macro_fragment;
   bool has_changed_flag;
 
   AST::MacroRulesDefinition *last_def;
diff --git a/gcc/rust/expand/rust-proc-macro-invoc-lexer.cc b/gcc/rust/expand/rust-proc-macro-invoc-lexer.cc
new file mode 100644
index 0000000..f365549
--- /dev/null
+++ b/gcc/rust/expand/rust-proc-macro-invoc-lexer.cc
@@ -0,0 +1,57 @@
+// Copyright (C) 2020-2023 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-proc-macro-invoc-lexer.h"
+#include "rust-token.h"
+
+namespace Rust {
+
+const_TokenPtr
+ProcMacroInvocLexer::peek_token (int n)
+{
+  if ((offs + n) >= token_stream.size ())
+    return Token::make (END_OF_FILE, Location ());
+
+  return token_stream.at (offs + n);
+}
+
+// Advances current token to n + 1 tokens ahead of current position.
+void
+ProcMacroInvocLexer::skip_token (int n)
+{
+  offs += (n + 1);
+}
+
+void
+ProcMacroInvocLexer::split_current_token (TokenId new_left, TokenId new_right)
+{
+  auto &current_token = token_stream.at (offs);
+  auto current_pos = token_stream.begin () + offs;
+
+  auto l_tok = Token::make (new_left, current_token->get_locus ());
+  auto r_tok = Token::make (new_right, current_token->get_locus ());
+
+  token_stream.erase (current_pos);
+
+  // `insert` inserts before the specified position, so we insert the right one
+  // then the left
+  token_stream.insert (current_pos, l_tok);
+  token_stream.insert (current_pos, r_tok);
+}
+
+} // namespace Rust
diff --git a/gcc/rust/expand/rust-proc-macro-invoc-lexer.h b/gcc/rust/expand/rust-proc-macro-invoc-lexer.h
new file mode 100644
index 0000000..7c047ef
--- /dev/null
+++ b/gcc/rust/expand/rust-proc-macro-invoc-lexer.h
@@ -0,0 +1,64 @@
+// Copyright (C) 2020-2023 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_PROC_MACRO_INVOC_LEXER_H
+#define RUST_PROC_MACRO_INVOC_LEXER_H
+
+#include "rust-lex.h"
+
+namespace Rust {
+class ProcMacroInvocLexer
+{
+public:
+  ProcMacroInvocLexer (std::vector<const_TokenPtr> stream)
+    : offs (0), token_stream (std::move (stream))
+  {}
+
+  // Returns token n tokens ahead of current position.
+  const_TokenPtr peek_token (int n);
+
+  // Peeks the current token.
+  const_TokenPtr peek_token () { return peek_token (0); }
+
+  // Advances current token to n + 1 tokens ahead of current position.
+  void skip_token (int n);
+
+  // Skips the current token.
+  void skip_token () { skip_token (0); }
+
+  // Splits the current token into two. Intended for use with nested generics
+  // closes (i.e. T<U<X>> where >> is wrongly lexed as one token). Note that
+  // this will only work with "simple" tokens like punctuation.
+  void split_current_token (TokenId new_left, TokenId new_right);
+
+  std::string get_filename () const
+  {
+    // FIXME
+    gcc_unreachable ();
+    return "FIXME";
+  }
+
+  size_t get_offs () const { return offs; }
+
+private:
+  size_t offs;
+  std::vector<const_TokenPtr> token_stream;
+};
+} // namespace Rust
+
+#endif /* ! RUST_PROC_MACRO_INVOC_LEXER_H */
diff --git a/gcc/rust/expand/rust-proc-macro.cc b/gcc/rust/expand/rust-proc-macro.cc
index a53a5d8..22744cb 100644
--- a/gcc/rust/expand/rust-proc-macro.cc
+++ b/gcc/rust/expand/rust-proc-macro.cc
@@ -61,47 +61,4 @@ load_macros (std::string path)
 					    array->macros + array->length);
 }
 
-void
-ProcMacroExpander::import_proc_macros (std::string extern_crate)
-{
-  auto path = session.extern_crates.find (extern_crate);
-  if (path == session.extern_crates.end ())
-    {
-      // Extern crate path is not available.
-      // FIXME: Emit error
-      rust_error_at (Location (), "Cannot find requested proc macro crate");
-      gcc_unreachable ();
-    }
-  auto macros = load_macros (path->second);
-
-  std::string prefix = extern_crate + "::";
-  for (auto &macro : macros)
-    {
-      switch (macro.tag)
-	{
-	case ProcMacro::CUSTOM_DERIVE:
-	  rust_debug ("Found one derive proc macro.");
-	  mappings->insert_derive_proc_macro (
-	    std::make_pair (extern_crate,
-			    macro.payload.custom_derive.trait_name),
-	    macro.payload.custom_derive);
-	  break;
-	case ProcMacro::ATTR:
-	  rust_debug ("Found one attribute proc macro.");
-	  mappings->insert_attribute_proc_macro (
-	    std::make_pair (extern_crate, macro.payload.attribute.name),
-	    macro.payload.attribute);
-	  break;
-	case ProcMacro::BANG:
-	  rust_debug ("Found one bang proc macro.");
-	  mappings->insert_bang_proc_macro (
-	    std::make_pair (extern_crate, macro.payload.bang.name),
-	    macro.payload.bang);
-	  break;
-	default:
-	  gcc_unreachable ();
-	}
-    }
-}
-
 } // namespace Rust
diff --git a/gcc/rust/expand/rust-proc-macro.h b/gcc/rust/expand/rust-proc-macro.h
index 244d274..7e9d7ec 100644
--- a/gcc/rust/expand/rust-proc-macro.h
+++ b/gcc/rust/expand/rust-proc-macro.h
@@ -17,97 +17,17 @@
 #ifndef RUST_PROC_MACRO_H
 #define RUST_PROC_MACRO_H
 
-#include <string>
-#include "rust-hir-map.h"
-#include "rust-name-resolver.h"
-#include "rust-session-manager.h"
-#include "rust-ast.h"
-#include "rust-ast-collector.h"
-#include "rust-token-converter.h"
 #include "libproc_macro/proc_macro.h"
 
 namespace Rust {
-
 /**
- * Load a procedural macro library and return a pointer to it's entrypoint.
+ * Load a procedural macro library and collect its entrypoints.
  *
  * @param The path to the shared object file to load.
  */
 const std::vector<ProcMacro::Procmacro>
 load_macros (std::string path);
 
-class ProcMacroExpander
-{
-public:
-  ProcMacroExpander (Session &session)
-    : session (session), has_changed_flag (false),
-      resolver (Resolver::Resolver::get ()),
-      mappings (Analysis::Mappings::get ())
-
-  {}
-
-  ~ProcMacroExpander () = default;
-
-  void import_proc_macros (std::string extern_crate);
-
-  template <typename T>
-  void expand_derive_proc_macro (T &item, std::string &trait_name)
-  {}
-
-  template <typename T>
-  void expand_bang_proc_macro (T &item, AST::SimplePath &path)
-  {}
-
-  template <typename T>
-  void expand_attribute_proc_macro (T &item, AST::SimplePath &path)
-  {
-    ProcMacro::Attribute macro;
-
-    std::string crate = path.get_segments ()[0].get_segment_name ();
-    std::string name = path.get_segments ()[1].get_segment_name ();
-    if (!mappings->lookup_attribute_proc_macro (std::make_pair (crate, name),
-						macro))
-      {
-	// FIXME: Resolve this path segment instead of taking it directly.
-	import_proc_macros (crate);
-      }
-
-    if (!mappings->lookup_attribute_proc_macro (std::make_pair (crate, name),
-						macro))
-      {
-	rust_error_at (Location (), "procedural macro %s not found",
-		       name.c_str ());
-	rust_assert (false);
-      }
-    // FIXME: Attach result back to the ast
-    std::vector<TokenPtr> tokens;
-    AST::TokenCollector collector (tokens);
-
-    collector.visit (item);
-
-    std::vector<const_TokenPtr> vec;
-    for (auto i : collector.collect_tokens ())
-      {
-	vec.push_back (std::const_pointer_cast<Token> (i));
-      }
-
-    // FIXME: Handle attributes
-    macro.macro (ProcMacro::TokenStream::make_tokenstream (), convert (vec));
-  }
-
-  bool has_changed () const { return has_changed_flag; }
-
-  void reset_changed_state () { has_changed_flag = false; }
-
-private:
-  Session &session;
-  bool has_changed_flag;
-
-public:
-  Resolver::Resolver *resolver;
-  Analysis::Mappings *mappings;
-};
-
 } // namespace Rust
 
 #endif /* ! RUST_PROC_MACRO_H */
diff --git a/gcc/rust/rust-session-manager.cc b/gcc/rust/rust-session-manager.cc
index bbcc569..17f8465 100644
--- a/gcc/rust/rust-session-manager.cc
+++ b/gcc/rust/rust-session-manager.cc
@@ -867,13 +867,12 @@ Session::expansion (AST::Crate &crate)
   /* expand by calling cxtctxt object's monotonic_expander's expand_crate
    * method. */
   MacroExpander expander (crate, cfg, *this);
-  ProcMacroExpander proc_expander (*this);
 
   while (!fixed_point_reached && iterations < cfg.recursion_limit)
     {
       CfgStrip ().go (crate);
       Resolver::EarlyNameResolver ().go (crate);
-      ExpandVisitor (expander, proc_expander).go (crate);
+      ExpandVisitor (expander).go (crate);
 
       fixed_point_reached = !expander.has_changed ();
       expander.reset_changed_state ();