preprocessor: Reimplement directives only processing, support raw literals.

The existing directives-only code (a) punched a hole through the
libcpp interface and (b) didn't support raw string literals.  This
reimplements this preprocessing mode.  I added a proper callback
interface, and adjusted c-ppoutput to use it.  Sadly I cannot get rid
of the libcpp/internal.h include for unrelated reasons.

The new scanner is in lex.x, and works doing some backwards scanning
when it finds a charater of interest.  This reduces the number of
cases one has to deal with in forward scanning.  It may have different
failure mode than forward scanning on bad tokenization.

Finally, Moved some cpp tests from the c-specific dg.gcc/cpp directory
to the c-c++-common/cpp shared directory,

	libcpp/
	* directives-only.c: Delete.
	* Makefile.in (libcpp_a_OBJS, libcpp_a_SOURCES): Remove it.
	* include/cpplib.h (enum CPP_DO_task): New enum.
	(cpp_directive_only_preprocess): Declare.
	* internal.h (_cpp_dir_only_callbacks): Delete.
	(_cpp_preprocess_dir_only): Delete.
	* lex.c (do_peek_backslask, do_peek_next, do_peek_prev): New.
	(cpp_directives_only_process): New implementation.

	gcc/c-family/
	Reimplement directives only processing.
	* c-ppoutput.c (token_streamer): Ne.
	(directives_only_cb): New.  Swallow ...
	(print_lines_directives_only): ... this.
	(scan_translation_unit_directives_only): Reimplment using the
	published interface.

	gcc/testsuite/
	* gcc.dg/cpp/counter-[23].c: Move to c-c+_-common/cpp.
	* gcc.dg/cpp/dir-only-*: Likewise.
	* c-c++-common/cpp/dir-only-[78].c: New.

16 files changed, 262 insertions, 12 deletions

diff --git a/gcc/c-family/ChangeLog b/gcc/c-family/ChangeLog
index 711da91..0550e53 100644
--- a/gcc/c-family/ChangeLog
+++ b/gcc/c-family/ChangeLog
@@ -1,3 +1,12 @@
+2020-05-08  Nathan Sidwell  <nathan@acm.org>
+
+	Reimplement directives only processing.
+	* c-ppoutput.c (token_streamer): Ne.
+	(directives_only_cb): New.  Swallow ...
+	(print_lines_directives_only): ... this.
+	(scan_translation_unit_directives_only): Reimplment using the
+	published interface.
+
 2020-05-07  Marek Polacek  <polacek@redhat.com>
 
 	* c-format.c (badwords): Add "nonstatic".
diff --git a/gcc/c-family/c-ppoutput.c b/gcc/c-family/c-ppoutput.c
index 57e7ae0..1e2b32b 100644
--- a/gcc/c-family/c-ppoutput.c
+++ b/gcc/c-family/c-ppoutput.c
@@ -52,7 +52,6 @@ static macro_queue *define_queue, *undef_queue;
 
 /* General output routines.  */
 static void scan_translation_unit (cpp_reader *);
-static void print_lines_directives_only (int, const void *, size_t);
 static void scan_translation_unit_directives_only (cpp_reader *);
 static void scan_translation_unit_trad (cpp_reader *);
 static void account_for_newlines (const unsigned char *, size_t);
@@ -165,6 +164,141 @@ init_pp_output (FILE *out_stream)
   print.prev_was_system_token = false;
 }
 
+// FIXME: Ideally we'd just turn the entirety of the print struct into
+// an encapsulated streamer ...
+
+class token_streamer
+{
+  bool avoid_paste;
+  bool do_line_adjustments;
+  bool in_pragma;
+  bool line_marker_emitted;
+
+ public:
+  token_streamer (cpp_reader *pfile)
+    :avoid_paste (false),
+    do_line_adjustments (cpp_get_options (pfile)->lang != CLK_ASM
+			 && !flag_no_line_commands),
+    in_pragma (false),
+    line_marker_emitted (false)
+    {
+    }
+
+  void begin_pragma () 
+  {
+    in_pragma = true;
+  }
+
+  void stream (cpp_reader *pfile, const cpp_token *tok, location_t);
+};
+
+void
+token_streamer::stream (cpp_reader *pfile, const cpp_token *token,
+			location_t loc)
+{
+  if (token->type == CPP_PADDING)
+    {
+      avoid_paste = true;
+      if (print.source == NULL
+	  || (!(print.source->flags & PREV_WHITE)
+	      && token->val.source == NULL))
+	print.source = token->val.source;
+      return;
+    }
+
+  if (token->type == CPP_EOF)
+    return;
+
+  /* Subtle logic to output a space if and only if necessary.  */
+  if (avoid_paste)
+    {
+      int src_line = LOCATION_LINE (loc);
+
+      if (print.source == NULL)
+	print.source = token;
+
+      if (src_line != print.src_line
+	  && do_line_adjustments
+	  && !in_pragma)
+	{
+	  line_marker_emitted = do_line_change (pfile, token, loc, false);
+	  putc (' ', print.outf);
+	  print.printed = true;
+	}
+      else if (print.source->flags & PREV_WHITE
+	       || (print.prev
+		   && cpp_avoid_paste (pfile, print.prev, token))
+	       || (print.prev == NULL && token->type == CPP_HASH))
+	{
+	  putc (' ', print.outf);
+	  print.printed = true;
+	}
+    }
+  else if (token->flags & PREV_WHITE)
+    {
+      int src_line = LOCATION_LINE (loc);
+
+      if (src_line != print.src_line
+	  && do_line_adjustments
+	  && !in_pragma)
+	line_marker_emitted = do_line_change (pfile, token, loc, false);
+      putc (' ', print.outf);
+      print.printed = true;
+    }
+
+  avoid_paste = false;
+  print.source = NULL;
+  print.prev = token;
+  if (token->type == CPP_PRAGMA)
+    {
+      const char *space;
+      const char *name;
+
+      line_marker_emitted = maybe_print_line (token->src_loc);
+      fputs ("#pragma ", print.outf);
+      c_pp_lookup_pragma (token->val.pragma, &space, &name);
+      if (space)
+	fprintf (print.outf, "%s %s", space, name);
+      else
+	fprintf (print.outf, "%s", name);
+      print.printed = true;
+      in_pragma = true;
+    }
+  else if (token->type == CPP_PRAGMA_EOL)
+    {
+      maybe_print_line (UNKNOWN_LOCATION);
+      in_pragma = false;
+    }
+  else
+    {
+      if (cpp_get_options (parse_in)->debug)
+	linemap_dump_location (line_table, token->src_loc, print.outf);
+
+      if (do_line_adjustments
+	  && !in_pragma
+	  && !line_marker_emitted
+	  && print.prev_was_system_token != !!in_system_header_at (loc)
+	  && !is_location_from_builtin_token (loc))
+	/* The system-ness of this token is different from the one of
+	   the previous token.  Let's emit a line change to mark the
+	   new system-ness before we emit the token.  */
+	{
+	  do_line_change (pfile, token, loc, false);
+	  print.prev_was_system_token = !!in_system_header_at (loc);
+	}
+      cpp_output_token (token, print.outf);
+      line_marker_emitted = false;
+      print.printed = true;
+    }
+
+  /* CPP_COMMENT tokens and raw-string literal tokens can have
+     embedded new-line characters.  Rather than enumerating all the
+     possible token types just check if token uses val.str union
+     member.  */
+  if (cpp_token_val_index (token) == CPP_TOKEN_FLD_STR)
+    account_for_newlines (token->val.str.text, token->val.str.len);
+}
+
 /* Writes out the preprocessed file, handling spacing and paste
    avoidance issues.  */
 static void
@@ -288,10 +422,41 @@ scan_translation_unit (cpp_reader *pfile)
 }
 
 static void
-print_lines_directives_only (int lines, const void *buf, size_t size)
+directives_only_cb (cpp_reader *pfile, CPP_DO_task task, void *data_, ...)
 {
-  print.src_line += lines;
-  fwrite (buf, 1, size, print.outf);
+  va_list args;
+  va_start (args, data_);
+
+  token_streamer *streamer = reinterpret_cast <token_streamer *> (data_);
+  switch (task)
+    {
+    default:
+      gcc_unreachable ();
+
+    case CPP_DO_print:
+      {
+	print.src_line += va_arg (args, unsigned);
+
+	const void *buf = va_arg (args, const void *);
+	size_t size = va_arg (args, size_t);
+	fwrite (buf, 1, size, print.outf);
+      }
+      break;
+
+    case CPP_DO_location:
+      maybe_print_line (va_arg (args, location_t));
+      break;
+
+    case CPP_DO_token:
+      {
+	const cpp_token *token = va_arg (args, const cpp_token *);
+	location_t spelling_loc = va_arg (args, location_t);
+	streamer->stream (pfile, token, spelling_loc);
+      }
+      break;
+    }
+
+  va_end (args);
 }
 
 /* Writes out the preprocessed file, handling spacing and paste
@@ -299,12 +464,8 @@ print_lines_directives_only (int lines, const void *buf, size_t size)
 static void
 scan_translation_unit_directives_only (cpp_reader *pfile)
 {
-  struct _cpp_dir_only_callbacks cb;
-
-  cb.print_lines = print_lines_directives_only;
-  cb.maybe_print_line = maybe_print_line;
-
-  _cpp_preprocess_dir_only (pfile, &cb);
+  token_streamer streamer (pfile);
+  cpp_directive_only_process (pfile, &streamer, directives_only_cb);
 }
 
 /* Adjust print.src_line for newlines embedded in output.  */
@@ -396,7 +557,7 @@ print_line_1 (location_t src_loc, const char *special_flags, FILE *stream)
     putc ('\n', stream);
   print.printed = false;
 
-  if (!flag_no_line_commands)
+  if (src_loc != UNKNOWN_LOCATION && !flag_no_line_commands)
     {
       const char *file_path = LOCATION_FILE (src_loc);
       int sysp;
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 9618eaa..2381a1c 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,9 @@
+2020-05-08  Nathan Sidwell  <nathan@acm.org>
+
+	* gcc.dg/cpp/counter-[23].c: Move to c-c+_-common/cpp.
+	* gcc.dg/cpp/dir-only-*: Likewise.
+	* c-c++-common/cpp/dir-only-[78].c: New.
+
 2020-05-08  Richard Biener  <rguenther@suse.de>
 
 	* gcc.dg/vect/bb-slp-pr68892.c: Adjust for not supported
diff --git a/gcc/testsuite/gcc.dg/cpp/counter-2.c b/gcc/testsuite/c-c++-common/cpp/counter-2.c
index 7d6578d..7d6578d 100644
--- a/gcc/testsuite/gcc.dg/cpp/counter-2.c
+++ b/gcc/testsuite/c-c++-common/cpp/counter-2.c
diff --git a/gcc/testsuite/gcc.dg/cpp/counter-3.c b/gcc/testsuite/c-c++-common/cpp/counter-3.c
index 3b1824f..3b1824f 100644
--- a/gcc/testsuite/gcc.dg/cpp/counter-3.c
+++ b/gcc/testsuite/c-c++-common/cpp/counter-3.c
diff --git a/gcc/testsuite/gcc.dg/cpp/dir-only-1.c b/gcc/testsuite/c-c++-common/cpp/dir-only-1.c
index 3c22616..3c22616 100644
--- a/gcc/testsuite/gcc.dg/cpp/dir-only-1.c
+++ b/gcc/testsuite/c-c++-common/cpp/dir-only-1.c
diff --git a/gcc/testsuite/gcc.dg/cpp/dir-only-1.h b/gcc/testsuite/c-c++-common/cpp/dir-only-1.h
index 96dbcc0..96dbcc0 100644
--- a/gcc/testsuite/gcc.dg/cpp/dir-only-1.h
+++ b/gcc/testsuite/c-c++-common/cpp/dir-only-1.h
diff --git a/gcc/testsuite/gcc.dg/cpp/dir-only-2.c b/gcc/testsuite/c-c++-common/cpp/dir-only-2.c
index 489b4d6..489b4d6 100644
--- a/gcc/testsuite/gcc.dg/cpp/dir-only-2.c
+++ b/gcc/testsuite/c-c++-common/cpp/dir-only-2.c
diff --git a/gcc/testsuite/gcc.dg/cpp/dir-only-3.c b/gcc/testsuite/c-c++-common/cpp/dir-only-3.c
index e6eaa01..e6eaa01 100644
--- a/gcc/testsuite/gcc.dg/cpp/dir-only-3.c
+++ b/gcc/testsuite/c-c++-common/cpp/dir-only-3.c
diff --git a/gcc/testsuite/gcc.dg/cpp/dir-only-3a.h b/gcc/testsuite/c-c++-common/cpp/dir-only-3a.h
index 6644bbf..6644bbf 100644
--- a/gcc/testsuite/gcc.dg/cpp/dir-only-3a.h
+++ b/gcc/testsuite/c-c++-common/cpp/dir-only-3a.h
diff --git a/gcc/testsuite/gcc.dg/cpp/dir-only-3b.h b/gcc/testsuite/c-c++-common/cpp/dir-only-3b.h
index 4edaa7b..4edaa7b 100644
--- a/gcc/testsuite/gcc.dg/cpp/dir-only-3b.h
+++ b/gcc/testsuite/c-c++-common/cpp/dir-only-3b.h
diff --git a/gcc/testsuite/gcc.dg/cpp/dir-only-4.c b/gcc/testsuite/c-c++-common/cpp/dir-only-4.c
index 25f1b46..25f1b46 100644
--- a/gcc/testsuite/gcc.dg/cpp/dir-only-4.c
+++ b/gcc/testsuite/c-c++-common/cpp/dir-only-4.c
diff --git a/gcc/testsuite/gcc.dg/cpp/dir-only-5.c b/gcc/testsuite/c-c++-common/cpp/dir-only-5.c
index ad8e11e..747006a 100644
--- a/gcc/testsuite/gcc.dg/cpp/dir-only-5.c
+++ b/gcc/testsuite/c-c++-common/cpp/dir-only-5.c
@@ -1,6 +1,7 @@
 /* Copyright 2007 Free Software Foundation, Inc.
    Contributed by Ollie Wild <aaw@google.com>.  */
 
-/* { dg-do preprocess } */
+/* C++ silently ignores traditional! */
+/* { dg-do preprocess { target c } } */
 /* { dg-options "-fdirectives-only -traditional" } */
 /* { dg-error "'-fdirectives-only' is incompatible with '-traditional'\n" "'-traditional' check" { target *-*-* } 0 } */
diff --git a/gcc/testsuite/gcc.dg/cpp/dir-only-6.c b/gcc/testsuite/c-c++-common/cpp/dir-only-6.c
index 0023205..0023205 100644
--- a/gcc/testsuite/gcc.dg/cpp/dir-only-6.c
+++ b/gcc/testsuite/c-c++-common/cpp/dir-only-6.c
diff --git a/gcc/testsuite/c-c++-common/cpp/dir-only-7.c b/gcc/testsuite/c-c++-common/cpp/dir-only-7.c
new file mode 100644
index 0000000..b9fb4d8
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/dir-only-7.c
@@ -0,0 +1,35 @@
+// { dg-do preprocess  }
+// { dg-options "-std=c++11" { target c++ } }
+// { dg-options "-std=gnu99" { target c } }
+// { dg-additional-options -fdirectives-only }
+
+R"stuff(
+)nope"
+#error in raw literal
+)stuff"
+// comment
+#define bob 1
+// " comment
+#if !bob
+#error "no bob"
+#endif
+
+bob\
+\
+R"regular string not an erroneous raw one"
+
+"regular"R"***(not a raw string"
+#define HERE 1
+ //)***"
+#ifndef HERE
+#error "oops no HERE"
+#endif
+ /* comment */
+
+
+0e+R"*(not a raw string"
+#define CPP_NUM 1
+ //)*"
+#ifndef CPP_NUM
+#error "oops no CPP_NUM"
+#endif
diff --git a/gcc/testsuite/c-c++-common/cpp/dir-only-8.c b/gcc/testsuite/c-c++-common/cpp/dir-only-8.c
new file mode 100644
index 0000000..b0e00f5
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/dir-only-8.c
@@ -0,0 +1,38 @@
+// { dg-do preprocess { target c++ } }
+// { dg-options "-std=c++14" }
+// { dg-additional-options -fdirectives-only }
+
+012'bcd
+#define A 1
+// '
+#ifndef A
+#error Fell into first char const
+#endif
+enum { A = 195'936'478 }; 'a'
+#define AA 1
+			  // 'a
+#ifndef AA
+#error Fell into early char const
+#endif
+
+012\
+'bcd
+#define B 1
+// '
+#ifndef B
+#error Fell into second char const
+#endif
+
+.012'b
+#define C 1
+// '
+#ifndef C
+#error Fell into third char const
+#endif
+
+.0e+12'b
+#define D 1
+// '
+#ifndef D
+#error Fell into fourth char const
+#endif