Improve lexer dump

2 files changed, 54 insertions, 5 deletions

diff --git a/gcc/rust/lex/rust-lex.cc b/gcc/rust/lex/rust-lex.cc
index 692af5d..ea17ecc 100644
--- a/gcc/rust/lex/rust-lex.cc
+++ b/gcc/rust/lex/rust-lex.cc
@@ -118,13 +118,15 @@ is_non_decimal_int_literal_separator (char character)
 
 Lexer::Lexer (const std::string &input)
   : input (RAIIFile::create_error ()), current_line (1), current_column (1),
-    line_map (nullptr), raw_input_source (new BufferInputSource (input, 0)),
+    line_map (nullptr), dump_lex_out (Optional<std::ofstream &>::none ()),
+    raw_input_source (new BufferInputSource (input, 0)),
     input_queue{*raw_input_source}, token_queue (TokenSource (this))
 {}
 
-Lexer::Lexer (const char *filename, RAIIFile file_input, Linemap *linemap)
+Lexer::Lexer (const char *filename, RAIIFile file_input, Linemap *linemap,
+	      Optional<std::ofstream &> dump_lex_opt)
   : input (std::move (file_input)), current_line (1), current_column (1),
-    line_map (linemap),
+    line_map (linemap), dump_lex_out (dump_lex_opt),
     raw_input_source (new FileInputSource (input.get_raw ())),
     input_queue{*raw_input_source}, token_queue (TokenSource (this))
 {
@@ -187,6 +189,45 @@ Lexer::skip_input ()
 }
 
 void
+Lexer::skip_token (int n)
+{
+  // dump tokens if dump-lex option is enabled
+  if (dump_lex_out.is_some ())
+    dump_and_skip (n);
+  else
+    token_queue.skip (n);
+}
+
+void
+Lexer::dump_and_skip (int n)
+{
+  std::ofstream &out = dump_lex_out.get ();
+  bool found_eof = false;
+  const_TokenPtr tok;
+  for (int i = 0; i < n + 1; i++)
+    {
+      if (!found_eof)
+	{
+	  tok = peek_token ();
+	  found_eof |= tok->get_id () == Rust::END_OF_FILE;
+
+	  Location loc = tok->get_locus ();
+
+	  out << "<id=";
+	  out << tok->token_id_to_str ();
+	  out << (tok->has_str () ? (std::string (", text=") + tok->get_str ()
+				     + std::string (", typehint=")
+				     + std::string (tok->get_type_hint_str ()))
+				  : "")
+	      << " ";
+	  out << get_line_map ()->to_string (loc) << " ";
+	}
+
+      token_queue.skip (0);
+    }
+}
+
+void
 Lexer::replace_current_token (TokenPtr replacement)
 {
   token_queue.replace_current_value (replacement);
diff --git a/gcc/rust/lex/rust-lex.h b/gcc/rust/lex/rust-lex.h
index 27120d1..c05e267 100644
--- a/gcc/rust/lex/rust-lex.h
+++ b/gcc/rust/lex/rust-lex.h
@@ -22,6 +22,7 @@
 #include "rust-linemap.h"
 #include "rust-buffered-queue.h"
 #include "rust-token.h"
+#include "rust-optional.h"
 
 namespace Rust {
 // Simple wrapper for FILE* that simplifies destruction.
@@ -139,7 +140,9 @@ private:
 
 public:
   // Construct lexer with input file and filename provided
-  Lexer (const char *filename, RAIIFile input, Linemap *linemap);
+  Lexer (const char *filename, RAIIFile input, Linemap *linemap,
+	 Optional<std::ofstream &> dump_lex_opt
+	 = Optional<std::ofstream &>::none ());
 
   // Lex the contents of a string instead of a file
   Lexer (const std::string &input);
@@ -161,10 +164,13 @@ public:
   const_TokenPtr peek_token () { return peek_token (0); }
 
   // Advances current token to n + 1 tokens ahead of current position.
-  void skip_token (int n) { token_queue.skip (n); }
+  void skip_token (int n);
   // Skips the current token.
   void skip_token () { skip_token (0); }
 
+  // Dumps and advances by n + 1 tokens.
+  void dump_and_skip (int n);
+
   // Replaces the current token with a specified token.
   void replace_current_token (TokenPtr replacement);
   // FIXME: don't use anymore
@@ -197,6 +203,8 @@ private:
    * allocating new linemap */
   static const int max_column_hint = 80;
 
+  Optional<std::ofstream &> dump_lex_out;
+
   // Input source wrapper thing.
   class InputSource
   {