lex: Add source code token string representation

Add a new representation for tokens which should reflect the string
token as it could be found in the original source.

gcc/rust/ChangeLog:

	* lex/rust-token.cc (Token::as_string): Add as_string
	implementation.
	* lex/rust-token.h: Add as_string prototype.

Signed-off-by: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>

2 files changed, 17 insertions, 0 deletions

diff --git a/gcc/rust/lex/rust-token.cc b/gcc/rust/lex/rust-token.cc
index 5a7dad6..a6be8473 100644
--- a/gcc/rust/lex/rust-token.cc
+++ b/gcc/rust/lex/rust-token.cc
@@ -131,4 +131,17 @@ Token::get_str () const
     }
   return *str;
 }
+
+std::string
+Token::as_string () const
+{
+  if (should_have_str ())
+    {
+      return get_str ();
+    }
+  else
+    {
+      return get_token_description ();
+    }
+}
 } // namespace Rust
diff --git a/gcc/rust/lex/rust-token.h b/gcc/rust/lex/rust-token.h
index f00d9cf..39c21cd 100644
--- a/gcc/rust/lex/rust-token.h
+++ b/gcc/rust/lex/rust-token.h
@@ -443,6 +443,10 @@ return *str;
 
   // Returns whether the token is a pure decimal int literal
   bool is_pure_decimal () const { return type_hint == CORETYPE_PURE_DECIMAL; }
+
+  // Return the token representation as someone would find it in the original
+  // source code file.
+  std::string as_string () const;
 };
 } // namespace Rust