gccrs: tokenstream: Convert group markers

Add conversion for group delimiters.

gcc/rust/ChangeLog:

	* ast/rust-ast-tokenstream.cc (pop_group): Add a function to
	easily collect a group from a given stack at the end of it.
	(TokenStream::collect): Collect tokens as a rust compatible
	Tokenstream type.
	* ast/rust-ast-tokenstream.h (RUST_AST_TOKENSTREAM_H): Move
	includes to stay constrained by guards.

Signed-off-by: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>

2 files changed, 48 insertions, 2 deletions

diff --git a/gcc/rust/ast/rust-ast-tokenstream.cc b/gcc/rust/ast/rust-ast-tokenstream.cc
index bd0b6e4..417c0f3 100644
--- a/gcc/rust/ast/rust-ast-tokenstream.cc
+++ b/gcc/rust/ast/rust-ast-tokenstream.cc
@@ -16,6 +16,7 @@
 // along with GCC; see the file COPYING3.  If not see
 // <http://www.gnu.org/licenses/>.
 #include "rust-ast-tokenstream.h"
+#include "libproc_macro/proc_macro.h"
 
 namespace Rust {
 namespace AST {
@@ -29,6 +30,47 @@ TokenStream::collect_tokens () const
   return tokens;
 }
 
+static void
+pop_group (std::vector<ProcMacro::TokenStream> &streams,
+	   ProcMacro::Delimiter delim)
+{
+  auto g = ProcMacro::Group::make_group (streams.back (), delim);
+  streams.pop_back ();
+  auto tt = ProcMacro::TokenTree::make_tokentree (g);
+
+  streams.back ().push (tt);
+}
+
+ProcMacro::TokenStream
+TokenStream::collect () const
+{
+  std::vector<ProcMacro::TokenStream> trees;
+  trees.push_back (ProcMacro::TokenStream::make_tokenstream ());
+  for (auto &token : collect_tokens ())
+    {
+      switch (token->get_id ())
+	{
+	case RIGHT_PAREN:
+	  pop_group (trees, ProcMacro::PARENTHESIS);
+	  break;
+	case RIGHT_CURLY:
+	  pop_group (trees, ProcMacro::BRACE);
+	  break;
+	case RIGHT_SQUARE:
+	  pop_group (trees, ProcMacro::BRACKET);
+	  break;
+	case LEFT_SQUARE:
+	case LEFT_CURLY:
+	case LEFT_PAREN:
+	  trees.push_back (ProcMacro::TokenStream::make_tokenstream ());
+	  break;
+	default:
+	  gcc_unreachable ();
+	}
+    }
+  return trees.back ();
+}
+
 void
 TokenStream::visit (AST::Crate &crate)
 {
diff --git a/gcc/rust/ast/rust-ast-tokenstream.h b/gcc/rust/ast/rust-ast-tokenstream.h
index 24be831..cf0f490 100644
--- a/gcc/rust/ast/rust-ast-tokenstream.h
+++ b/gcc/rust/ast/rust-ast-tokenstream.h
@@ -16,13 +16,15 @@
 // along with GCC; see the file COPYING3.  If not see
 // <http://www.gnu.org/licenses/>.
 
+#ifndef RUST_AST_TOKENSTREAM_H
+#define RUST_AST_TOKENSTREAM_H
+
 #include "rust-token.h"
 #include "rust-ast-visitor.h"
 #include "rust-ast.h"
 #include "rust-ast-full.h"
 
-#ifndef RUST_AST_TOKENSTREAM_H
-#define RUST_AST_TOKENSTREAM_H
+#include "libproc_macro/tokenstream.h"
 
 namespace Rust {
 namespace AST {
@@ -38,6 +40,8 @@ public:
 
   std::vector<TokenPtr> collect_tokens () const;
 
+  ProcMacro::TokenStream collect () const;
+
 private:
   std::vector<TokenPtr> &tokens;