Merge #956

956: Substitute repetitions r=CohenArthur a=CohenArthur

Needs #955 

This PR splits up the `substitute_tokens` function into multiple smaller functions. Still a draft until I can get repetitions working.

Closes #960 
Closes #961 

Co-authored-by: Arthur Cohen <arthur.cohen@embecosm.com>

1 files changed, 96 insertions, 13 deletions

diff --git a/gcc/rust/expand/rust-macro-expand.h b/gcc/rust/expand/rust-macro-expand.h
index edb091d..eeafdb8 100644
--- a/gcc/rust/expand/rust-macro-expand.h
+++ b/gcc/rust/expand/rust-macro-expand.h
@@ -57,11 +57,21 @@ struct MatchedFragment
   size_t match_amount;
 
   MatchedFragment (std::string identifier, size_t token_offset_begin,
-		   size_t token_offset_end, size_t match_amount = 0)
+		   size_t token_offset_end, size_t match_amount = 1)
     : fragment_ident (identifier), token_offset_begin (token_offset_begin),
       token_offset_end (token_offset_end), match_amount (match_amount)
   {}
 
+  /**
+   * Create a valid fragment matched zero times. This is useful for repetitions
+   * which allow the absence of a fragment, such as * and ?
+   */
+  static MatchedFragment zero (std::string identifier)
+  {
+    // We don't need offsets since there is "no match"
+    return MatchedFragment (identifier, 0, 0, 0);
+  }
+
   std::string as_string () const
   {
     return fragment_ident + "=" + std::to_string (token_offset_begin) + ":"
@@ -79,17 +89,38 @@ public:
 
   void push () { stack.push_back ({}); }
 
-  std::map<std::string, MatchedFragment> pop ()
+  std::map<std::string, std::vector<MatchedFragment>> pop ()
   {
     auto top = stack.back ();
     stack.pop_back ();
     return top;
   }
 
-  std::map<std::string, MatchedFragment> &peek () { return stack.back (); }
+  std::map<std::string, std::vector<MatchedFragment>> &peek ()
+  {
+    return stack.back ();
+  }
+
+  void insert_fragment (MatchedFragment fragment)
+  {
+    auto &current_map = stack.back ();
+    auto it = current_map.find (fragment.fragment_ident);
+
+    if (it == current_map.end ())
+      {
+	auto new_frags = std::vector<MatchedFragment> ();
+	new_frags.emplace_back (fragment);
+	current_map.insert ({fragment.fragment_ident, new_frags});
+      }
+    else
+      {
+	auto &frags = it->second;
+	frags.emplace_back (fragment);
+      }
+  }
 
 private:
-  std::vector<std::map<std::string, MatchedFragment>> stack;
+  std::vector<std::map<std::string, std::vector<MatchedFragment>>> stack;
 };
 
 // Object used to store shared data (between functions) for macro expansion.
@@ -141,11 +172,10 @@ struct MacroExpander
   bool try_match_rule (AST::MacroRule &match_rule,
 		       AST::DelimTokenTree &invoc_token_tree);
 
-  AST::ASTFragment
-  transcribe_rule (AST::MacroRule &match_rule,
-		   AST::DelimTokenTree &invoc_token_tree,
-		   std::map<std::string, MatchedFragment> &matched_fragments,
-		   bool semicolon, ContextType ctx);
+  AST::ASTFragment transcribe_rule (
+    AST::MacroRule &match_rule, AST::DelimTokenTree &invoc_token_tree,
+    std::map<std::string, std::vector<MatchedFragment>> &matched_fragments,
+    bool semicolon, ContextType ctx);
 
   bool match_fragment (Parser<MacroInvocLexer> &parser,
 		       AST::MacroMatchFragment &fragment);
@@ -183,10 +213,63 @@ struct MacroExpander
 			size_t &match_amount, size_t lo_bound = 0,
 			size_t hi_bound = 0);
 
-  static std::vector<std::unique_ptr<AST::Token>>
-  substitute_tokens (std::vector<std::unique_ptr<AST::Token>> &input,
-		     std::vector<std::unique_ptr<AST::Token>> &macro,
-		     std::map<std::string, MatchedFragment> &fragments);
+  /**
+   * Substitute a metavariable by its given fragment in a transcribing context,
+   * i.e. replacing $var with the associated fragment.
+   *
+   * @param input Tokens given to the transcribing context
+   * @param fragments Fragments given to the macro substitution
+   * @param metavar Metavariable to try and replace
+   *
+   * @return A token containing the associated fragment expanded into tokens if
+   * any, or the cloned token if no fragment was associated
+   */
+  static std::vector<std::unique_ptr<AST::Token>> substitute_metavar (
+    std::vector<std::unique_ptr<AST::Token>> &input,
+    std::map<std::string, std::vector<MatchedFragment>> &fragments,
+    std::unique_ptr<AST::Token> &metavar);
+
+  /**
+   * Substitute a macro repetition by its given fragments
+   *
+   * @param input Tokens given to the transcribing context
+   * @param fragments Fragments given to the macro substitution
+   * @param pattern_start Start index of the pattern tokens
+   * @param pattern_end Index  Amount of tokens in the pattern
+   *
+   * @return A vector containing the repeated pattern
+   */
+  static std::vector<std::unique_ptr<AST::Token>> substitute_repetition (
+    std::vector<std::unique_ptr<AST::Token>> &input,
+    std::vector<std::unique_ptr<AST::Token>> &macro,
+    std::map<std::string, std::vector<MatchedFragment>> &fragments,
+    size_t pattern_start, size_t pattern_end);
+
+  /**
+   * Substitute a given token by its appropriate representation
+   *
+   * @param macro Tokens used in the macro declaration
+   * @param input Tokens given to the transcribing context
+   * @param fragments Fragments given to the macro substitution
+   * @param token Current token to try and substitute
+   *
+   * @return A token containing the associated fragment expanded into tokens if
+   * any, or the cloned token if no fragment was associated, as well as the
+   * amount of tokens that should be skipped before the next invocation. Since
+   * this function may consume more than just one token, it is important to skip
+   * ahead of the input to avoid mis-substitutions
+   */
+  static std::pair<std::vector<std::unique_ptr<AST::Token>>, size_t>
+  substitute_token (
+    std::vector<std::unique_ptr<AST::Token>> &input,
+    std::vector<std::unique_ptr<AST::Token>> &macro,
+    std::map<std::string, std::vector<MatchedFragment>> &fragments,
+    size_t token_idx);
+
+  static std::vector<std::unique_ptr<AST::Token>> substitute_tokens (
+    std::vector<std::unique_ptr<AST::Token>> &input,
+    std::vector<std::unique_ptr<AST::Token>> &macro,
+    std::map<std::string, std::vector<MatchedFragment>> &fragments);
 
   void push_context (ContextType t) { context.push_back (t); }