Merge #1607

1607: Improve AST Fragment class r=CohenArthur a=CohenArthur

This changes the APIs around creating AST fragments and refactors the class into its own header and source file, hopefully making it easier to use. This will also help creating "unexpanded" AST fragments for proper builtin macro expansion with the new fixed-point algorithm introduced by #1606 

`@liushuyu` pinging you since you've worked extensively with the macro system. Would love your review!

Co-authored-by: Arthur Cohen <arthur.cohen@embecosm.com>

4 files changed, 298 insertions, 141 deletions

diff --git a/gcc/rust/ast/rust-ast-fragment.cc b/gcc/rust/ast/rust-ast-fragment.cc
new file mode 100644
index 0000000..c491609
--- /dev/null
+++ b/gcc/rust/ast/rust-ast-fragment.cc
@@ -0,0 +1,171 @@
+// Copyright (C) 2020-2022 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-ast-fragment.h"
+
+namespace Rust {
+namespace AST {
+
+Fragment::Fragment (FragmentKind kind, std::vector<SingleASTNode> nodes)
+  : kind (kind), nodes (std::move (nodes))
+{}
+
+Fragment::Fragment (Fragment const &other) : kind (other.get_kind ())
+{
+  *this = other;
+}
+
+Fragment &
+Fragment::operator= (Fragment const &other)
+{
+  nodes.clear ();
+  nodes.reserve (other.nodes.size ());
+  kind = other.get_kind ();
+  for (auto &n : other.nodes)
+    {
+      nodes.push_back (n);
+    }
+
+  return *this;
+}
+
+Fragment
+Fragment::create_error ()
+{
+  return Fragment (FragmentKind::Error, {});
+}
+
+Fragment
+Fragment::complete (std::vector<AST::SingleASTNode> nodes)
+{
+  return Fragment (FragmentKind::Complete, std::move (nodes));
+}
+
+Fragment
+Fragment::unexpanded ()
+{
+  return Fragment (FragmentKind::Unexpanded, {});
+}
+
+std::vector<SingleASTNode> &
+Fragment::get_nodes ()
+{
+  return nodes;
+}
+
+FragmentKind
+Fragment::get_kind () const
+{
+  return kind;
+}
+
+bool
+Fragment::is_error () const
+{
+  return get_kind () == FragmentKind::Error;
+}
+
+bool
+Fragment::should_expand () const
+{
+  return !is_error ();
+}
+
+bool
+Fragment::is_expression_fragment () const
+{
+  return is_single_fragment_of_kind (SingleASTNode::NodeType::EXPRESSION);
+}
+
+bool
+Fragment::is_type_fragment () const
+{
+  return is_single_fragment_of_kind (SingleASTNode::NodeType::TYPE);
+}
+
+std::unique_ptr<Expr>
+Fragment::take_expression_fragment ()
+{
+  assert_single_fragment (SingleASTNode::NodeType::EXPRESSION);
+  return nodes[0].take_expr ();
+}
+
+std::unique_ptr<Type>
+Fragment::take_type_fragment ()
+{
+  assert_single_fragment (SingleASTNode::NodeType::TYPE);
+  return nodes[0].take_type ();
+}
+
+void
+Fragment::accept_vis (ASTVisitor &vis)
+{
+  for (auto &node : nodes)
+    node.accept_vis (vis);
+}
+
+bool
+Fragment::is_single_fragment () const
+{
+  return nodes.size () == 1;
+}
+
+bool
+Fragment::is_single_fragment_of_kind (SingleASTNode::NodeType expected) const
+{
+  return is_single_fragment () && nodes[0].get_kind () == expected;
+}
+
+void
+Fragment::assert_single_fragment (SingleASTNode::NodeType expected) const
+{
+  static const std::map<SingleASTNode::NodeType, const char *> str_map = {
+    {SingleASTNode::NodeType::IMPL, "impl"},
+    {SingleASTNode::NodeType::ITEM, "item"},
+    {SingleASTNode::NodeType::TYPE, "type"},
+    {SingleASTNode::NodeType::EXPRESSION, "expr"},
+    {SingleASTNode::NodeType::STMT, "stmt"},
+    {SingleASTNode::NodeType::EXTERN, "extern"},
+    {SingleASTNode::NodeType::TRAIT, "trait"},
+    {SingleASTNode::NodeType::TRAIT_IMPL, "trait impl"},
+  };
+
+  auto actual = nodes[0].get_kind ();
+  auto fail = false;
+
+  if (!is_single_fragment ())
+    {
+      rust_error_at (Location (), "fragment is not single");
+      fail = true;
+    }
+
+  if (actual != expected)
+    {
+      rust_error_at (
+	Location (),
+	"invalid fragment operation: expected %qs node, got %qs node",
+	str_map.find (expected)->second,
+	str_map.find (nodes[0].get_kind ())->second);
+      fail = true;
+    }
+
+  rust_assert (!fail);
+}
+
+} // namespace AST
+} // namespace Rust
diff --git a/gcc/rust/ast/rust-ast-fragment.h b/gcc/rust/ast/rust-ast-fragment.h
new file mode 100644
index 0000000..3ef4ba1
--- /dev/null
+++ b/gcc/rust/ast/rust-ast-fragment.h
@@ -0,0 +1,118 @@
+// Copyright (C) 2020-2022 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_AST_FRAGMENT_H
+#define RUST_AST_FRAGMENT_H
+
+#include "rust-ast.h"
+#include "rust-system.h"
+
+namespace Rust {
+namespace AST {
+
+enum class FragmentKind
+{
+  /**
+   * If an AST Fragment still contains unexpanded tokens - this should only be
+   * used in the case of builtin macros which need to be expanded eagerly.
+   */
+  Unexpanded,
+  /**
+   * A completely expanded AST Fragment. This signifies that all
+   * `SingleASTNode`s in the `nodes` vector are valid.
+   *
+   * Note that this doesn't imply that the expansion is "done". One of the
+   * expanded nodes could very well be another macro invocation
+   */
+  Complete,
+  /**
+   * An error fragment.
+   */
+  Error,
+};
+
+/**
+ * An AST Fragment. Previously named `ASTFragment`.
+ *
+ * Basically, a "fragment" that can be incorporated into the AST, created as
+ * a result of macro expansion. Really annoying to work with due to the fact
+ * that macros can really expand to anything. As such, horrible representation
+ * at the moment.
+ */
+class Fragment
+{
+public:
+  Fragment (Fragment const &other);
+  Fragment &operator= (Fragment const &other);
+
+  /**
+   * Create an error fragment
+   */
+  static Fragment create_error ();
+
+  /**
+   * Create a complete AST fragment
+   */
+  static Fragment complete (std::vector<AST::SingleASTNode> nodes);
+
+  /**
+   * Create a fragment which contains unexpanded nodes
+   */
+  static Fragment unexpanded ();
+
+  FragmentKind get_kind () const;
+  std::vector<SingleASTNode> &get_nodes ();
+
+  bool is_error () const;
+  bool should_expand () const;
+
+  bool is_expression_fragment () const;
+  bool is_type_fragment () const;
+
+  std::unique_ptr<Expr> take_expression_fragment ();
+  std::unique_ptr<Type> take_type_fragment ();
+
+  void accept_vis (ASTVisitor &vis);
+
+private:
+  Fragment (FragmentKind kind, std::vector<SingleASTNode> nodes);
+
+  FragmentKind kind;
+
+  /**
+   * Basic idea: essentially, a vector of tagged unions of different AST node
+   * types. Now, this could actually be stored without a tagged union if the
+   * different AST node types had a unified parent, but that would create
+   * issues with the diamond problem or significant performance penalties. So
+   * a tagged union had to be used instead. A vector is used to represent the
+   * ability for a macro to expand to two statements, for instance.
+   */
+  std::vector<SingleASTNode> nodes;
+
+  /**
+   * We need to make a special case for Expression and Type fragments as only
+   * one Node will be extracted from the `nodes` vector
+   */
+  bool is_single_fragment () const;
+  bool is_single_fragment_of_kind (SingleASTNode::NodeType expected) const;
+  void assert_single_fragment (SingleASTNode::NodeType expected) const;
+};
+} // namespace AST
+} // namespace Rust
+
+#endif // !RUST_AST_FRAGMENT_H
diff --git a/gcc/rust/ast/rust-ast.h b/gcc/rust/ast/rust-ast.h
index 492faea..e0e10dc 100644
--- a/gcc/rust/ast/rust-ast.h
+++ b/gcc/rust/ast/rust-ast.h
@@ -1858,138 +1858,6 @@ public:
   }
 };
 
-/* Basically, a "fragment" that can be incorporated into the AST, created as
- * a result of macro expansion. Really annoying to work with due to the fact
- * that macros can really expand to anything. As such, horrible representation
- * at the moment. */
-class ASTFragment
-{
-private:
-  /* basic idea: essentially, a vector of tagged unions of different AST node
-   * types. Now, this could actually be stored without a tagged union if the
-   * different AST node types had a unified parent, but that would create
-   * issues with the diamond problem or significant performance penalties. So
-   * a tagged union had to be used instead. A vector is used to represent the
-   * ability for a macro to expand to two statements, for instance. */
-
-  std::vector<SingleASTNode> nodes;
-  bool fragment_is_error;
-
-  /**
-   * We need to make a special case for Expression and Type fragments as only
-   * one Node will be extracted from the `nodes` vector
-   */
-
-  bool is_single_fragment () const { return nodes.size () == 1; }
-
-  bool is_single_fragment_of_kind (SingleASTNode::NodeType expected) const
-  {
-    return is_single_fragment () && nodes[0].get_kind () == expected;
-  }
-
-  void assert_single_fragment (SingleASTNode::NodeType expected) const
-  {
-    static const std::map<SingleASTNode::NodeType, const char *> str_map = {
-      {SingleASTNode::NodeType::IMPL, "impl"},
-      {SingleASTNode::NodeType::ITEM, "item"},
-      {SingleASTNode::NodeType::TYPE, "type"},
-      {SingleASTNode::NodeType::EXPRESSION, "expr"},
-      {SingleASTNode::NodeType::STMT, "stmt"},
-      {SingleASTNode::NodeType::EXTERN, "extern"},
-      {SingleASTNode::NodeType::TRAIT, "trait"},
-      {SingleASTNode::NodeType::TRAIT_IMPL, "trait impl"},
-    };
-
-    auto actual = nodes[0].get_kind ();
-    auto fail = false;
-
-    if (!is_single_fragment ())
-      {
-	rust_error_at (Location (), "fragment is not single");
-	fail = true;
-      }
-
-    if (actual != expected)
-      {
-	rust_error_at (
-	  Location (),
-	  "invalid fragment operation: expected %qs node, got %qs node",
-	  str_map.find (expected)->second,
-	  str_map.find (nodes[0].get_kind ())->second);
-	fail = true;
-      }
-
-    rust_assert (!fail);
-  }
-
-public:
-  ASTFragment (std::vector<SingleASTNode> nodes, bool fragment_is_error = false)
-    : nodes (std::move (nodes)), fragment_is_error (fragment_is_error)
-  {
-    if (fragment_is_error)
-      rust_assert (nodes.empty ());
-  }
-
-  ASTFragment (ASTFragment const &other)
-    : fragment_is_error (other.fragment_is_error)
-  {
-    nodes.clear ();
-    nodes.reserve (other.nodes.size ());
-    for (auto &n : other.nodes)
-      {
-	nodes.push_back (n);
-      }
-  }
-
-  ASTFragment &operator= (ASTFragment const &other)
-  {
-    fragment_is_error = other.fragment_is_error;
-    nodes.clear ();
-    nodes.reserve (other.nodes.size ());
-    for (auto &n : other.nodes)
-      {
-	nodes.push_back (n);
-      }
-
-    return *this;
-  }
-
-  static ASTFragment create_error () { return ASTFragment ({}, true); }
-
-  std::vector<SingleASTNode> &get_nodes () { return nodes; }
-  bool is_error () const { return fragment_is_error; }
-
-  bool should_expand () const { return !is_error (); }
-
-  bool is_expression_fragment () const
-  {
-    return is_single_fragment_of_kind (SingleASTNode::NodeType::EXPRESSION);
-  }
-
-  bool is_type_fragment () const
-  {
-    return is_single_fragment_of_kind (SingleASTNode::NodeType::TYPE);
-  }
-
-  std::unique_ptr<Expr> take_expression_fragment ()
-  {
-    assert_single_fragment (SingleASTNode::NodeType::EXPRESSION);
-    return nodes[0].take_expr ();
-  }
-
-  std::unique_ptr<Type> take_type_fragment ()
-  {
-    assert_single_fragment (SingleASTNode::NodeType::TYPE);
-    return nodes[0].take_type ();
-  }
-
-  void accept_vis (ASTVisitor &vis)
-  {
-    for (auto &node : nodes)
-      node.accept_vis (vis);
-  }
-};
-
 // A crate AST object - holds all the data for a single compilation unit
 struct Crate
 {
diff --git a/gcc/rust/ast/rust-macro.h b/gcc/rust/ast/rust-macro.h
index 276f441..a02c216 100644
--- a/gcc/rust/ast/rust-macro.h
+++ b/gcc/rust/ast/rust-macro.h
@@ -20,6 +20,7 @@
 #define RUST_AST_MACRO_H
 
 #include "rust-ast.h"
+#include "rust-ast-fragment.h"
 #include "rust-location.h"
 #include <string>
 
@@ -456,8 +457,7 @@ class MacroRulesDefinition : public MacroItem
   std::vector<MacroRule> rules; // inlined form
   Location locus;
 
-  std::function<ASTFragment (Location, MacroInvocData &)>
-    associated_transcriber;
+  std::function<Fragment (Location, MacroInvocData &)> associated_transcriber;
   // Since we can't compare std::functions, we need to use an extra boolean
   bool is_builtin_rule;
 
@@ -468,10 +468,10 @@ class MacroRulesDefinition : public MacroItem
    * should make use of the actual rules. If the macro is builtin, then another
    * associated transcriber should be used
    */
-  static ASTFragment dummy_builtin (Location, MacroInvocData &)
+  static Fragment dummy_builtin (Location, MacroInvocData &)
   {
     gcc_unreachable ();
-    return ASTFragment::create_error ();
+    return Fragment::create_error ();
   }
 
   /* NOTE: in rustc, macro definitions are considered (and parsed as) a type
@@ -491,9 +491,9 @@ public:
       associated_transcriber (dummy_builtin), is_builtin_rule (false)
   {}
 
-  MacroRulesDefinition (Identifier builtin_name, DelimType delim_type,
-			std::function<ASTFragment (Location, MacroInvocData &)>
-			  associated_transcriber)
+  MacroRulesDefinition (
+    Identifier builtin_name, DelimType delim_type,
+    std::function<Fragment (Location, MacroInvocData &)> associated_transcriber)
     : outer_attrs (std::vector<Attribute> ()), rule_name (builtin_name),
       delim_type (delim_type), rules (std::vector<MacroRule> ()),
       locus (Location ()), associated_transcriber (associated_transcriber),
@@ -521,14 +521,14 @@ public:
   const std::vector<MacroRule> &get_rules () const { return rules; }
 
   bool is_builtin () const { return is_builtin_rule; }
-  const std::function<ASTFragment (Location, MacroInvocData &)> &
+  const std::function<Fragment (Location, MacroInvocData &)> &
   get_builtin_transcriber () const
   {
     rust_assert (is_builtin ());
     return associated_transcriber;
   }
   void set_builtin_transcriber (
-    std::function<ASTFragment (Location, MacroInvocData &)> transcriber)
+    std::function<Fragment (Location, MacroInvocData &)> transcriber)
   {
     associated_transcriber = transcriber;
     is_builtin_rule = true;