Unified Tokens partially in preparation for macro expansion

5 files changed, 63 insertions, 179 deletions

diff --git a/gcc/rust/ast/rust-ast-full-test.cc b/gcc/rust/ast/rust-ast-full-test.cc
index e1308b7..2c196a0 100644
--- a/gcc/rust/ast/rust-ast-full-test.cc
+++ b/gcc/rust/ast/rust-ast-full-test.cc
@@ -290,13 +290,17 @@ DelimTokenTree::as_string () const
 std::string
 Token::as_string () const
 {
-  /* FIXME: only works when not identifier or literal or whatever, i.e. when
-   * doesn't store string value */
-  // return get_token_description(token_id);
+  if (tok_ref->has_str ())
+    {
+      std::string str = tok_ref->get_str ();
 
-  // maybe fixed - stores everything as string though, so storage-inefficient
-  std::string quote = is_string_lit () ? "\"" : "";
-  return quote + str + quote;
+      std::string quote = is_string_lit () ? "\"" : "";
+      return quote + str + quote;
+    }
+  else
+    {
+      return tok_ref->get_token_description ();
+    }
 }
 
 std::string
@@ -4375,9 +4379,10 @@ DelimTokenTree::to_token_stream () const
   std::vector<std::unique_ptr<Token> > tokens;
 
   // simulate presence of delimiters
+  const_TokenPtr left_paren
+    = Rust::Token::make (LEFT_PAREN, Linemap::unknown_location ());
   tokens.push_back (
-    std::unique_ptr<Token> (new Token (LEFT_PAREN, Linemap::unknown_location (),
-				       "", CORETYPE_UNKNOWN)));
+    std::unique_ptr<Token> (new Token (std::move (left_paren))));
 
   for (const auto &tree : token_trees)
     {
@@ -4387,9 +4392,10 @@ DelimTokenTree::to_token_stream () const
 		     std::make_move_iterator (stream.end ()));
     }
 
-  tokens.push_back (std::unique_ptr<Token> (
-    new Token (RIGHT_PAREN, Linemap::unknown_location (), "",
-	       CORETYPE_UNKNOWN)));
+  const_TokenPtr right_paren
+    = Rust::Token::make (RIGHT_PAREN, Linemap::unknown_location ());
+  tokens.push_back (
+    std::unique_ptr<Token> (new Token (std::move (right_paren))));
 
   tokens.shrink_to_fit ();
 
@@ -4545,8 +4551,7 @@ AttrInputMetaItemContainer::check_cfg_predicate (const Session &session) const
 }
 
 bool
-MetaItemLitExpr::check_cfg_predicate (
-  const Session &session ATTRIBUTE_UNUSED) const
+MetaItemLitExpr::check_cfg_predicate (const Session &) const
 {
   /* as far as I can tell, a literal expr can never be a valid cfg body, so
    * false */
@@ -4653,10 +4658,7 @@ bool
 MetaListPaths::check_path_exists_in_cfg (const Session &session,
 					 const SimplePath &path) const
 {
-  auto it = session.options.target_data.features.find (path.as_string ());
-  if (it != session.options.target_data.features.end ())
-    return true;
-  return false;
+  return session.options.target_data.has_key (path.as_string ());
 }
 
 bool
@@ -4710,10 +4712,7 @@ MetaItemSeq::check_cfg_predicate (const Session &session) const
 bool
 MetaWord::check_cfg_predicate (const Session &session) const
 {
-  auto it = session.options.target_data.features.find (ident);
-  if (it != session.options.target_data.features.end ())
-    return true;
-  return false;
+  return session.options.target_data.has_key (ident);
 }
 
 bool
@@ -4723,10 +4722,7 @@ MetaItemPath::check_cfg_predicate (const Session &session) const
    * relating to SimplePath being identifier. Currently, it would return true
    * if path as identifier existed, and if the path in string form existed
    * (though this shouldn't occur). */
-  auto it = session.options.target_data.features.find (path.as_string ());
-  if (it != session.options.target_data.features.end ())
-    return true;
-  return false;
+  return session.options.target_data.has_key (path.as_string ());
 }
 
 bool
@@ -4853,8 +4849,8 @@ AttrInputMetaItemContainer::separate_cfg_attrs () const
       Attribute attr = (*it)->to_attribute ();
       if (attr.is_empty ())
 	{
-	  // TODO should this be an error that causes us to chuck out
-	  // everything?
+	  /* TODO should this be an error that causes us to chuck out
+	   * everything? */
 	  continue;
 	}
       attrs.push_back (std::move (attr));
diff --git a/gcc/rust/ast/rust-ast.h b/gcc/rust/ast/rust-ast.h
index b80e7a3..14816e2 100644
--- a/gcc/rust/ast/rust-ast.h
+++ b/gcc/rust/ast/rust-ast.h
@@ -129,6 +129,7 @@ class Token : public TokenTree, public MacroMatch
 {
   // A token is a kind of token tree (except delimiter tokens)
   // A token is a kind of MacroMatch (except $ and delimiter tokens)
+#if 0
   // TODO: improve member variables - current ones are the same as lexer token
   // Token kind.
   TokenId token_id;
@@ -138,6 +139,13 @@ class Token : public TokenTree, public MacroMatch
   std::string str;
   // Token type hint (if any).
   PrimitiveCoreType type_hint;
+#endif
+
+  const_TokenPtr tok_ref;
+
+  /* new idea: wrapper around const_TokenPtr used for heterogeneuous storage in
+   * token trees. rather than convert back and forth when parsing macros, just 
+   * wrap it. */
 
 public:
   // Unique pointer custom clone function
@@ -146,6 +154,7 @@ public:
     return std::unique_ptr<Token> (clone_token_impl ());
   }
 
+#if 0
   /* constructor from general text - avoid using if lexer const_TokenPtr is
    * available */
   Token (TokenId token_id, Location locus, std::string str,
@@ -153,8 +162,11 @@ public:
     : token_id (token_id), locus (locus), str (std::move (str)),
       type_hint (type_hint)
   {}
+#endif
+  // not doable with new implementation - will have to make a const_TokenPtr
 
   // Constructor from lexer const_TokenPtr
+#if 0
   /* TODO: find workaround for std::string being nullptr - probably have to
    * introduce new method in lexer Token, or maybe make conversion method
    * there */
@@ -188,10 +200,12 @@ public:
 		 lexer_token_ptr->get_token_description ());
       }
   }
+#endif
+  Token (const_TokenPtr lexer_tok_ptr) : tok_ref (std::move (lexer_tok_ptr)) {}
 
   bool is_string_lit () const
   {
-    switch (token_id)
+    switch (get_id ())
       {
       case STRING_LITERAL:
       case BYTE_STRING_LITERAL:
@@ -208,11 +222,14 @@ public:
   // Return copy of itself but in token stream form.
   std::vector<std::unique_ptr<Token> > to_token_stream () const override;
 
-  TokenId get_id () const { return token_id; }
+  TokenId get_id () const { return tok_ref->get_id (); }
 
-  Location get_locus () const { return locus; }
+  Location get_locus () const { return tok_ref->get_locus (); }
 
-  PrimitiveCoreType get_type_hint () const { return type_hint; }
+  PrimitiveCoreType get_type_hint () const { return tok_ref->get_type_hint (); }
+
+  // Get a new token pointer copy.
+  const_TokenPtr get_tok_ptr () const { return tok_ref; }
 
 protected:
   // No virtual for now as not polymorphic but can be in future
@@ -1190,15 +1207,8 @@ protected:
 class LifetimeParam : public GenericParam
 {
   Lifetime lifetime;
-
-  // bool has_lifetime_bounds;
-  // LifetimeBounds lifetime_bounds;
-  std::vector<Lifetime> lifetime_bounds; // inlined LifetimeBounds
-
-  // bool has_outer_attribute;
-  // std::unique_ptr<Attribute> outer_attr;
+  std::vector<Lifetime> lifetime_bounds; 
   Attribute outer_attr;
-
   Location locus;
 
 public:
@@ -1211,45 +1221,21 @@ public:
   // Creates an error state lifetime param.
   static LifetimeParam create_error ()
   {
-    return LifetimeParam (Lifetime::error ());
+    return LifetimeParam (Lifetime::error (), {}, Attribute::create_empty (), Location ());
   }
 
   // Returns whether the lifetime param is in an error state.
   bool is_error () const { return lifetime.is_error (); }
 
   // Constructor
-  LifetimeParam (Lifetime lifetime, Location locus = Location (),
-		 std::vector<Lifetime> lifetime_bounds
-		 = std::vector<Lifetime> (),
-		 Attribute outer_attr = Attribute::create_empty ())
+  LifetimeParam (Lifetime lifetime, 
+		 std::vector<Lifetime> lifetime_bounds,
+		 Attribute outer_attr, Location locus)
     : lifetime (std::move (lifetime)),
       lifetime_bounds (std::move (lifetime_bounds)),
       outer_attr (std::move (outer_attr)), locus (locus)
   {}
 
-  // TODO: remove copy and assignment operator definitions - not required
-
-  // Copy constructor with clone
-  LifetimeParam (LifetimeParam const &other)
-    : lifetime (other.lifetime), lifetime_bounds (other.lifetime_bounds),
-      outer_attr (other.outer_attr), locus (other.locus)
-  {}
-
-  // Overloaded assignment operator to clone attribute
-  LifetimeParam &operator= (LifetimeParam const &other)
-  {
-    lifetime = other.lifetime;
-    lifetime_bounds = other.lifetime_bounds;
-    outer_attr = other.outer_attr;
-    locus = other.locus;
-
-    return *this;
-  }
-
-  // move constructors
-  LifetimeParam (LifetimeParam &&other) = default;
-  LifetimeParam &operator= (LifetimeParam &&other) = default;
-
   std::string as_string () const override;
 
   void accept_vis (ASTVisitor &vis) override;
@@ -1263,28 +1249,13 @@ protected:
   }
 };
 
-// A macro item AST node - potentially abstract base class
+// A macro item AST node - abstract base class
 class MacroItem : public Item
-{
-  /*public:
-  std::string as_string() const;*/
-  // std::vector<Attribute> outer_attrs;
-
-protected:
-  /*MacroItem (std::vector<Attribute> outer_attribs)
-    : outer_attrs (std::move (outer_attribs))
-  {}*/
-};
+{};
 
 // Item used in trait declarations - abstract base class
 class TraitItem
 {
-  // bool has_outer_attrs;
-  // TODO: remove and rely on virtual functions and VisItem-derived attributes?
-  // std::vector<Attribute> outer_attrs;
-
-  // NOTE: all children should have outer attributes
-
 protected:
   // Clone function implementation as pure virtual method
   virtual TraitItem *clone_trait_item_impl () const = 0;
@@ -1468,69 +1439,18 @@ class MacroInvocationSemi : public MacroItem,
 			    public ExternalItem
 {
   std::vector<Attribute> outer_attrs;
-#if 0
-  SimplePath path;
-  // all delim types except curly must have invocation end with a semicolon
-  DelimType delim_type;
-  std::vector<std::unique_ptr<TokenTree> > token_trees;
-#endif
   MacroInvocData invoc_data;
   Location locus;
 
 public:
   std::string as_string () const override;
 
-  /*MacroInvocationSemi (SimplePath macro_path, DelimType delim_type,
-		       std::vector<std::unique_ptr<TokenTree> > token_trees,
-		       std::vector<Attribute> outer_attribs, Location locus)
-    : outer_attrs (std::move (outer_attribs)), path (std::move (macro_path)),
-      delim_type (delim_type), token_trees (std::move (token_trees)),
-      locus (locus)
-  {}*/
   MacroInvocationSemi (MacroInvocData invoc_data,
 		       std::vector<Attribute> outer_attrs, Location locus)
     : outer_attrs (std::move (outer_attrs)),
       invoc_data (std::move (invoc_data)), locus (locus)
   {}
 
-  /*
-  // Copy constructor with vector clone
-  MacroInvocationSemi (MacroInvocationSemi const &other)
-    : MacroItem (other), TraitItem (other), InherentImplItem (other),
-      TraitImplItem (other), outer_attrs (other.outer_attrs), path (other.path),
-      delim_type (other.delim_type), locus (other.locus)
-  {
-    token_trees.reserve (other.token_trees.size ());
-    for (const auto &e : other.token_trees)
-      token_trees.push_back (e->clone_token_tree ());
-  }*/
-
-  /*
-  // Overloaded assignment operator to vector clone
-  MacroInvocationSemi &operator= (MacroInvocationSemi const &other)
-  {
-    MacroItem::operator= (other);
-    TraitItem::operator= (other);
-    InherentImplItem::operator= (other);
-    TraitImplItem::operator= (other);
-    outer_attrs = other.outer_attrs;
-    path = other.path;
-    delim_type = other.delim_type;
-    locus = other.locus;
-
-    token_trees.reserve (other.token_trees.size ());
-    for (const auto &e : other.token_trees)
-      token_trees.push_back (e->clone_token_tree ());
-
-    return *this;
-  }*/
-
-  /*
-  // Move constructors
-  MacroInvocationSemi (MacroInvocationSemi &&other) = default;
-  MacroInvocationSemi &operator= (MacroInvocationSemi &&other) = default;
-  */
-
   void accept_vis (ASTVisitor &vis) override;
 
   // Clones this macro invocation semi.
@@ -1540,11 +1460,6 @@ public:
       clone_macro_invocation_semi_impl ());
   }
 
-  /*
-  // Invalid if path is empty, so base stripping on that.
-  void mark_for_strip () override { path = SimplePath::create_empty (); }
-  bool is_marked_for_strip () const override { return path.is_empty (); }
-  */
   void mark_for_strip () override { invoc_data.mark_for_strip (); }
   bool is_marked_for_strip () const override
   {
diff --git a/gcc/rust/ast/rust-macro.h b/gcc/rust/ast/rust-macro.h
index 29a1cf1..51220e4 100644
--- a/gcc/rust/ast/rust-macro.h
+++ b/gcc/rust/ast/rust-macro.h
@@ -361,19 +361,12 @@ class MacroInvocation : public TypeNoBounds,
 			public ExprWithoutBlock
 {
   std::vector<Attribute> outer_attrs;
-  /*SimplePath path;
-  DelimTokenTree token_tree;*/
   MacroInvocData invoc_data;
   Location locus;
 
 public:
   std::string as_string () const override;
 
-  /*MacroInvocation (SimplePath path, DelimTokenTree token_tree,
-		   std::vector<Attribute> outer_attrs, Location locus)
-    : ExprWithoutBlock (std::move (outer_attrs)), path (std::move (path)),
-      token_tree (std::move (token_tree)), locus (locus)
-  {}*/
   MacroInvocation (MacroInvocData invoc_data,
 		   std::vector<Attribute> outer_attrs, Location locus)
     : outer_attrs (std::move (outer_attrs)),
@@ -694,11 +687,8 @@ public:
 struct MacroParser
 {
 private:
+  // TODO: might as well rewrite to use lexer tokens
   std::vector<std::unique_ptr<Token> > token_stream;
-  /* probably have to make this mutable (mutable int stream_pos) otherwise const
-   * has to be removed up to DelimTokenTree or further ok since this changing
-   * would have an effect on the results of the methods run (i.e. not logically
-   * const), the parsing methods shouldn't be const */
   int stream_pos;
 
 public:
diff --git a/gcc/rust/lex/rust-token.h b/gcc/rust/lex/rust-token.h
index fab736a..6070244 100644
--- a/gcc/rust/lex/rust-token.h
+++ b/gcc/rust/lex/rust-token.h
@@ -318,13 +318,6 @@ public:
   }
 
   // Makes and returns a new TokenPtr of type INT_LITERAL.
-  /*static TokenPtr make_int (Location locus, const std::string &str)
-  {
-    //return TokenPtr (new Token (INT_LITERAL, locus, str));
-    return std::make_shared<Token>(INT_LITERAL, locus, str);
-  }*/
-
-  // Makes and returns a new TokenPtr of type INT_LITERAL.
   static TokenPtr make_int (Location locus, const std::string &str,
 			    PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
   {
@@ -333,13 +326,6 @@ public:
   }
 
   // Makes and returns a new TokenPtr of type FLOAT_LITERAL.
-  /*static TokenPtr make_float (Location locus, const std::string &str)
-  {
-    return TokenPtr (new Token (FLOAT_LITERAL, locus, str));
-    return std::make_shared<Token>(FLOAT_LITERAL, locus, str);
-  }*/
-
-  // Makes and returns a new TokenPtr of type FLOAT_LITERAL.
   static TokenPtr make_float (Location locus, const std::string &str,
 			      PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
   {
diff --git a/gcc/rust/parse/rust-parse-impl.h b/gcc/rust/parse/rust-parse-impl.h
index cc30f1b..4a9f3a3 100644
--- a/gcc/rust/parse/rust-parse-impl.h
+++ b/gcc/rust/parse/rust-parse-impl.h
@@ -972,8 +972,7 @@ Parser<ManagedTokenSource>::parse_token_tree ()
     default:
       // parse token itself as TokenTree
       lexer.skip_token ();
-      // TODO: fix that token constructor, possibly with c++11 features
-      return std::unique_ptr<AST::Token> (new AST::Token (t));
+      return std::unique_ptr<AST::Token> (new AST::Token (std::move (t)));
     }
 }
 
@@ -1805,7 +1804,7 @@ Parser<ManagedTokenSource>::parse_macro_match ()
     default:
       // just the token
       lexer.skip_token ();
-      return std::unique_ptr<AST::Token> (new AST::Token (t));
+      return std::unique_ptr<AST::Token> (new AST::Token (std::move (t)));
     }
 }
 
@@ -1913,7 +1912,7 @@ Parser<ManagedTokenSource>::parse_macro_match_repetition ()
       break;
     default:
       // separator does exist
-      separator = std::unique_ptr<AST::Token> (new AST::Token (t));
+      separator = std::unique_ptr<AST::Token> (new AST::Token (std::move (t)));
       lexer.skip_token ();
       break;
     }
@@ -2665,9 +2664,9 @@ Parser<ManagedTokenSource>::parse_generic_params ()
 	}
 
       std::unique_ptr<AST::LifetimeParam> param (
-	new AST::LifetimeParam (std::move (lifetime), locus,
+	new AST::LifetimeParam (std::move (lifetime), 
 				std::move (lifetime_bounds),
-				std::move (outer_attr)));
+				std::move (outer_attr), locus));
       generic_params.push_back (std::move (param));
 
       if (lexer.peek_token ()->get_id () != COMMA)
@@ -2841,9 +2840,9 @@ Parser<ManagedTokenSource>::parse_generic_params (EndTokenPred is_end_token)
 	}
 
       std::unique_ptr<AST::LifetimeParam> param (
-	new AST::LifetimeParam (std::move (lifetime), locus,
+	new AST::LifetimeParam (std::move (lifetime),
 				std::move (lifetime_bounds),
-				std::move (outer_attr)));
+				std::move (outer_attr), locus));
       generic_params.push_back (std::move (param));
 
       if (lexer.peek_token ()->get_id () != COMMA)
@@ -3166,9 +3165,9 @@ Parser<ManagedTokenSource>::parse_lifetime_param ()
       // TODO: have end token passed in?
     }
 
-  return AST::LifetimeParam (std::move (lifetime), lifetime_tok->get_locus (),
+  return AST::LifetimeParam (std::move (lifetime), 
 			     std::move (lifetime_bounds),
-			     std::move (outer_attr));
+			     std::move (outer_attr), lifetime_tok->get_locus ());
 }
 
 // Parses type generic parameters. Will also consume any trailing comma.
@@ -3693,13 +3692,11 @@ Parser<ManagedTokenSource>::parse_trait_bound ()
       lexer.skip_token ();
     }
 
-  // parse for lifetimes, if it exists (although empty for lifetimes is ok to
-  // handle this)
+  /* parse for lifetimes, if it exists (although empty for lifetimes is ok to
+   * handle this) */
   std::vector<AST::LifetimeParam> for_lifetimes;
   if (lexer.peek_token ()->get_id () == FOR)
-    {
       for_lifetimes = parse_for_lifetimes ();
-    }
 
   // handle TypePath
   AST::TypePath type_path = parse_type_path ();