converter: Add punct conversion function

Add the implementation of the Punct conversion function to tokens.

gcc/rust/ChangeLog:

	* util/rust-token-converter.cc (from_punct): Add conversion
	implementation.

Signed-off-by: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>

1 files changed, 16 insertions, 1 deletions

diff --git a/gcc/rust/util/rust-token-converter.cc b/gcc/rust/util/rust-token-converter.cc
index 711c89d..8e289e9 100644
--- a/gcc/rust/util/rust-token-converter.cc
+++ b/gcc/rust/util/rust-token-converter.cc
@@ -14,6 +14,7 @@
 // along with GCC; see the file COPYING3.  If not see
 // <http://www.gnu.org/licenses/>.
 
+#include "rust-lex.h"
 #include "rust-token-converter.h"
 #include "libproc_macro/proc_macro.h"
 
@@ -310,13 +311,27 @@ from_literal (ProcMacro::Literal literal, std::vector<const_TokenPtr> &result)
 {}
 
 /**
+ * Accumulate through successive calls multiple Punct until one is tagged
+ * "Alone", then append the formed token to a given result vector.
  *
+ * @param punct Reference to the Punct to convert.
  * @param acc Reference to an accumulator for joined Punct.
+ * @param result Reference to the output token vector.
  */
 static void
 from_punct (const ProcMacro::Punct &punct, std::vector<std::uint32_t> &acc,
 	    std::vector<const_TokenPtr> &result)
-{}
+{
+  acc.push_back (punct.ch);
+  if (ProcMacro::ALONE == punct.spacing) /* Last punct of a chain */
+    {
+      // TODO: UTF-8 string
+      std::string whole (acc.begin (), acc.end ());
+      auto lexer = Lexer (whole);
+      result.push_back (lexer.peek_token ());
+      acc.clear ();
+    }
+}
 
 /**
  * Iterate over a Group and append all inner tokens to a vector enclosed by its