libproc_macro: Add remaining tokenstream structs.

Add remaining missing tokenstream structures. Most are interdependent.

ChangeLog:

	* libgrust/libproc_macro/Makefile.am: Add tokenstream,
	group and  tokentree source files.
	* libgrust/libproc_macro/Makefile.in: Regenerate.
	* libgrust/libproc_macro/group.cc: New file.
	* libgrust/libproc_macro/group.h: New file.
	* libgrust/libproc_macro/tokenstream.cc: New file.
	* libgrust/libproc_macro/tokenstream.h: New file.
	* libgrust/libproc_macro/tokentree.cc: New file.
	* libgrust/libproc_macro/tokentree.h: New file.

Signed-off-by: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>

8 files changed, 400 insertions, 2 deletions

diff --git a/libgrust/libproc_macro/Makefile.am b/libgrust/libproc_macro/Makefile.am
index 4d5cbbd..2846ac8 100644
--- a/libgrust/libproc_macro/Makefile.am
+++ b/libgrust/libproc_macro/Makefile.am
@@ -54,6 +54,9 @@ libproc_macro_la_SOURCES = \
 	proc_macro.cc \
 	ident.cc \
 	punct.cc \
+	group.cc \
+	tokentree.cc \
+	tokenstream.cc \
 	literal.cc
 
 include_HEADERS = \
diff --git a/libgrust/libproc_macro/Makefile.in b/libgrust/libproc_macro/Makefile.in
index efd332a..77fe662 100644
--- a/libgrust/libproc_macro/Makefile.in
+++ b/libgrust/libproc_macro/Makefile.in
@@ -138,8 +138,8 @@ am__installdirs = "$(DESTDIR)$(toolexeclibdir)" \
 	"$(DESTDIR)$(includedir)"
 LTLIBRARIES = $(toolexeclib_LTLIBRARIES)
 libproc_macro_la_LIBADD =
-am_libproc_macro_la_OBJECTS = proc_macro.lo ident.lo punct.lo \
-	literal.lo
+am_libproc_macro_la_OBJECTS = proc_macro.lo ident.lo punct.lo group.lo \
+	tokentree.lo tokenstream.lo literal.lo
 libproc_macro_la_OBJECTS = $(am_libproc_macro_la_OBJECTS)
 AM_V_lt = $(am__v_lt_@AM_V@)
 am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
@@ -389,6 +389,9 @@ libproc_macro_la_SOURCES = \
 	proc_macro.cc \
 	ident.cc \
 	punct.cc \
+	group.cc \
+	tokentree.cc \
+	tokenstream.cc \
 	literal.cc
 
 include_HEADERS = \
@@ -472,10 +475,13 @@ mostlyclean-compile:
 distclean-compile:
 	-rm -f *.tab.c
 
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/group.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ident.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/literal.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_macro.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/punct.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tokenstream.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tokentree.Plo@am__quote@
 
 .cc.o:
 @am__fastdepCXX_TRUE@	$(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
diff --git a/libgrust/libproc_macro/group.cc b/libgrust/libproc_macro/group.cc
new file mode 100644
index 0000000..e6fdce5
--- /dev/null
+++ b/libgrust/libproc_macro/group.cc
@@ -0,0 +1,33 @@
+// Copyright (C) 2023 Free Software Foundation, Inc.
+//
+// This file is part of the GNU Proc Macro Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#include "group.h"
+
+namespace Group {
+
+Group
+Group::make_group (TokenStream::TokenStream stream, Delimiter delim)
+{
+  return {delim, stream};
+}
+
+} // namespace Group
diff --git a/libgrust/libproc_macro/group.h b/libgrust/libproc_macro/group.h
new file mode 100644
index 0000000..09a7add
--- /dev/null
+++ b/libgrust/libproc_macro/group.h
@@ -0,0 +1,49 @@
+// Copyright (C) 2023 Free Software Foundation, Inc.
+//
+// This file is part of the GNU Proc Macro Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef GROUP_H
+#define GROUP_H
+
+#include "tokenstream.h"
+
+namespace Group {
+
+enum Delimiter
+{
+  Parenthesis,
+  Brace,
+  Bracket,
+  None,
+};
+
+struct Group
+{
+  Delimiter delimiter;
+  TokenStream::TokenStream stream;
+
+public:
+  static Group make_group (TokenStream::TokenStream stream, Delimiter delim);
+};
+
+} // namespace Group
+
+#endif /* ! GROUP_H */
diff --git a/libgrust/libproc_macro/tokenstream.cc b/libgrust/libproc_macro/tokenstream.cc
new file mode 100644
index 0000000..b8116c1
--- /dev/null
+++ b/libgrust/libproc_macro/tokenstream.cc
@@ -0,0 +1,110 @@
+// Copyright (C) 2023 Free Software Foundation, Inc.
+//
+// This file is part of the GNU Proc Macro Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#include "tokenstream.h"
+#include "tokentree.h"
+
+#include <cstring>
+
+namespace TokenStream {
+
+TokenStream
+TokenStream::make_tokenstream (std::vector<TokenTree::TokenTree> vec)
+{
+  auto stream = make_tokenstream (vec.size ());
+  for (auto tt : vec)
+    {
+      stream.push (tt);
+    }
+  return stream;
+}
+
+TokenStream
+TokenStream::make_tokenstream (std::uint64_t capacity)
+{
+  auto *data = new TokenTree::TokenTree[capacity];
+  return {data, 0, capacity};
+}
+
+void
+TokenStream::grow (std::uint64_t delta)
+{
+  auto new_capacity = capacity + delta;
+  auto *new_data = new TokenTree::TokenTree[new_capacity];
+  std::memcpy (new_data, data, size);
+  delete[] data;
+  data = new_data;
+}
+
+void
+TokenStream::push (TokenTree::TokenTree tree)
+{
+  if (size == capacity)
+    grow (capacity);
+  data[size] = tree;
+  size++;
+}
+
+extern "C" TokenStream
+TokenStream__new ()
+{
+  return TokenStream::make_tokenstream ();
+}
+
+extern "C" TokenStream
+TokenStream__with_capacity (std::uint64_t capacity)
+{
+  return TokenStream::make_tokenstream (capacity);
+}
+
+extern "C" void
+TokenSream__push (TokenStream *stream, TokenTree::TokenTree tree)
+{
+  stream->push (tree);
+}
+
+extern "C" bool
+TokenStream__from_string (unsigned char *str, std::uint64_t len,
+			  TokenStream *ts)
+{
+  // FIXME: Implement using parser ?
+  return false;
+}
+
+extern "C" TokenStream
+TokenStream__clone (const TokenStream *ts)
+{
+  auto *data = new TokenTree::TokenTree[ts->capacity];
+  std::memcpy (data, ts->data, ts->size);
+  return {data, ts->size, ts->capacity};
+}
+
+extern "C" void
+TokenStream__drop (TokenStream *stream)
+{
+  // FIXME: Also drop stream components
+  delete[] stream->data;
+  stream->capacity = 0;
+  stream->size = 0;
+}
+
+} // namespace TokenStream
diff --git a/libgrust/libproc_macro/tokenstream.h b/libgrust/libproc_macro/tokenstream.h
new file mode 100644
index 0000000..513553e
--- /dev/null
+++ b/libgrust/libproc_macro/tokenstream.h
@@ -0,0 +1,75 @@
+// Copyright (C) 2023 Free Software Foundation, Inc.
+//
+// This file is part of the GNU Proc Macro Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef TOKENSTREAM_H
+#define TOKENSTREAM_H
+
+#include <cstdint>
+#include <vector>
+
+namespace TokenTree {
+struct TokenTree;
+}
+
+namespace TokenStream {
+
+const std::int64_t DEFAULT_CAPACITY = 0;
+
+struct TokenStream
+{
+  TokenTree::TokenTree *data;
+  std::uint64_t size;
+  std::uint64_t capacity;
+
+public:
+  void grow (std::uint64_t delta);
+  void push (TokenTree::TokenTree tree);
+
+  TokenStream clone () const;
+
+  static TokenStream make_tokenstream (std::vector<TokenTree::TokenTree> vec);
+  static TokenStream make_tokenstream (std::uint64_t capacity
+				       = DEFAULT_CAPACITY);
+};
+
+extern "C" TokenStream
+TokenStream__new ();
+
+extern "C" TokenStream
+TokenStream__with_capacity (std::uint64_t capacity);
+
+extern "C" void
+TokenSream__push (TokenStream *stream, TokenTree::TokenTree tree);
+
+extern "C" bool
+TokenStream__from_string (unsigned char *str, std::uint64_t len,
+			  TokenStream *ts);
+
+extern "C" TokenStream
+TokenStream__clone (const TokenStream *ts);
+
+extern "C" void
+TokenStream__drop (TokenStream *stream);
+
+} // namespace TokenStream
+
+#endif /* ! TOKENSTREAM_H */
diff --git a/libgrust/libproc_macro/tokentree.cc b/libgrust/libproc_macro/tokentree.cc
new file mode 100644
index 0000000..fd9d981
--- /dev/null
+++ b/libgrust/libproc_macro/tokentree.cc
@@ -0,0 +1,59 @@
+// Copyright (C) 2023 Free Software Foundation, Inc.
+//
+// This file is part of the GNU Proc Macro Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#include "tokentree.h"
+
+namespace TokenTree {
+
+TokenTree
+TokenTree::make_tokentree (Group::Group group)
+{
+  TokenTreePayload payload;
+  payload.group = group;
+  return {GROUP, payload};
+}
+
+TokenTree
+TokenTree::make_tokentree (Ident::Ident ident)
+{
+  TokenTreePayload payload;
+  payload.ident = ident;
+  return {IDENT, payload};
+}
+
+TokenTree
+TokenTree::make_tokentree (Punct::Punct punct)
+{
+  TokenTreePayload payload;
+  payload.punct = punct;
+  return {PUNCT, payload};
+}
+
+TokenTree
+TokenTree::make_tokentree (Literal::Literal literal)
+{
+  TokenTreePayload payload;
+  payload.literal = literal;
+  return {LITERAL, payload};
+}
+
+} // namespace TokenTree
diff --git a/libgrust/libproc_macro/tokentree.h b/libgrust/libproc_macro/tokentree.h
new file mode 100644
index 0000000..a982bfc
--- /dev/null
+++ b/libgrust/libproc_macro/tokentree.h
@@ -0,0 +1,63 @@
+// Copyright (C) 2023 Free Software Foundation, Inc.
+//
+// This file is part of the GNU Proc Macro Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef TOKENTREE_H
+#define TOKENTREE_H
+
+#include "group.h"
+#include "ident.h"
+#include "punct.h"
+#include "literal.h"
+
+namespace TokenTree {
+
+enum TokenTreeTag
+{
+  GROUP,
+  IDENT,
+  PUNCT,
+  LITERAL
+};
+
+union TokenTreePayload
+{
+  Group::Group group;
+  Ident::Ident ident;
+  Punct::Punct punct;
+  Literal::Literal literal;
+};
+
+struct TokenTree
+{
+  TokenTreeTag tag;
+  TokenTreePayload payload;
+
+public:
+  static TokenTree make_tokentree (Group::Group group);
+  static TokenTree make_tokentree (Ident::Ident ident);
+  static TokenTree make_tokentree (Punct::Punct punct);
+  static TokenTree make_tokentree (Literal::Literal literal);
+};
+
+} // namespace TokenTree
+
+#endif /* ! TOKENTREE_H */