preprocessor: Reimplement directives only processing, support raw literals.

The existing directives-only code (a) punched a hole through the
libcpp interface and (b) didn't support raw string literals.  This
reimplements this preprocessing mode.  I added a proper callback
interface, and adjusted c-ppoutput to use it.  Sadly I cannot get rid
of the libcpp/internal.h include for unrelated reasons.

The new scanner is in lex.x, and works doing some backwards scanning
when it finds a charater of interest.  This reduces the number of
cases one has to deal with in forward scanning.  It may have different
failure mode than forward scanning on bad tokenization.

Finally, Moved some cpp tests from the c-specific dg.gcc/cpp directory
to the c-c++-common/cpp shared directory,

	libcpp/
	* directives-only.c: Delete.
	* Makefile.in (libcpp_a_OBJS, libcpp_a_SOURCES): Remove it.
	* include/cpplib.h (enum CPP_DO_task): New enum.
	(cpp_directive_only_preprocess): Declare.
	* internal.h (_cpp_dir_only_callbacks): Delete.
	(_cpp_preprocess_dir_only): Delete.
	* lex.c (do_peek_backslask, do_peek_next, do_peek_prev): New.
	(cpp_directives_only_process): New implementation.

	gcc/c-family/
	Reimplement directives only processing.
	* c-ppoutput.c (token_streamer): Ne.
	(directives_only_cb): New.  Swallow ...
	(print_lines_directives_only): ... this.
	(scan_translation_unit_directives_only): Reimplment using the
	published interface.

	gcc/testsuite/
	* gcc.dg/cpp/counter-[23].c: Move to c-c+_-common/cpp.
	* gcc.dg/cpp/dir-only-*: Likewise.
	* c-c++-common/cpp/dir-only-[78].c: New.

1 files changed, 172 insertions, 11 deletions

diff --git a/gcc/c-family/c-ppoutput.c b/gcc/c-family/c-ppoutput.c
index 57e7ae0..1e2b32b 100644
--- a/gcc/c-family/c-ppoutput.c
+++ b/gcc/c-family/c-ppoutput.c
@@ -52,7 +52,6 @@ static macro_queue *define_queue, *undef_queue;
 
 /* General output routines.  */
 static void scan_translation_unit (cpp_reader *);
-static void print_lines_directives_only (int, const void *, size_t);
 static void scan_translation_unit_directives_only (cpp_reader *);
 static void scan_translation_unit_trad (cpp_reader *);
 static void account_for_newlines (const unsigned char *, size_t);
@@ -165,6 +164,141 @@ init_pp_output (FILE *out_stream)
   print.prev_was_system_token = false;
 }
 
+// FIXME: Ideally we'd just turn the entirety of the print struct into
+// an encapsulated streamer ...
+
+class token_streamer
+{
+  bool avoid_paste;
+  bool do_line_adjustments;
+  bool in_pragma;
+  bool line_marker_emitted;
+
+ public:
+  token_streamer (cpp_reader *pfile)
+    :avoid_paste (false),
+    do_line_adjustments (cpp_get_options (pfile)->lang != CLK_ASM
+			 && !flag_no_line_commands),
+    in_pragma (false),
+    line_marker_emitted (false)
+    {
+    }
+
+  void begin_pragma () 
+  {
+    in_pragma = true;
+  }
+
+  void stream (cpp_reader *pfile, const cpp_token *tok, location_t);
+};
+
+void
+token_streamer::stream (cpp_reader *pfile, const cpp_token *token,
+			location_t loc)
+{
+  if (token->type == CPP_PADDING)
+    {
+      avoid_paste = true;
+      if (print.source == NULL
+	  || (!(print.source->flags & PREV_WHITE)
+	      && token->val.source == NULL))
+	print.source = token->val.source;
+      return;
+    }
+
+  if (token->type == CPP_EOF)
+    return;
+
+  /* Subtle logic to output a space if and only if necessary.  */
+  if (avoid_paste)
+    {
+      int src_line = LOCATION_LINE (loc);
+
+      if (print.source == NULL)
+	print.source = token;
+
+      if (src_line != print.src_line
+	  && do_line_adjustments
+	  && !in_pragma)
+	{
+	  line_marker_emitted = do_line_change (pfile, token, loc, false);
+	  putc (' ', print.outf);
+	  print.printed = true;
+	}
+      else if (print.source->flags & PREV_WHITE
+	       || (print.prev
+		   && cpp_avoid_paste (pfile, print.prev, token))
+	       || (print.prev == NULL && token->type == CPP_HASH))
+	{
+	  putc (' ', print.outf);
+	  print.printed = true;
+	}
+    }
+  else if (token->flags & PREV_WHITE)
+    {
+      int src_line = LOCATION_LINE (loc);
+
+      if (src_line != print.src_line
+	  && do_line_adjustments
+	  && !in_pragma)
+	line_marker_emitted = do_line_change (pfile, token, loc, false);
+      putc (' ', print.outf);
+      print.printed = true;
+    }
+
+  avoid_paste = false;
+  print.source = NULL;
+  print.prev = token;
+  if (token->type == CPP_PRAGMA)
+    {
+      const char *space;
+      const char *name;
+
+      line_marker_emitted = maybe_print_line (token->src_loc);
+      fputs ("#pragma ", print.outf);
+      c_pp_lookup_pragma (token->val.pragma, &space, &name);
+      if (space)
+	fprintf (print.outf, "%s %s", space, name);
+      else
+	fprintf (print.outf, "%s", name);
+      print.printed = true;
+      in_pragma = true;
+    }
+  else if (token->type == CPP_PRAGMA_EOL)
+    {
+      maybe_print_line (UNKNOWN_LOCATION);
+      in_pragma = false;
+    }
+  else
+    {
+      if (cpp_get_options (parse_in)->debug)
+	linemap_dump_location (line_table, token->src_loc, print.outf);
+
+      if (do_line_adjustments
+	  && !in_pragma
+	  && !line_marker_emitted
+	  && print.prev_was_system_token != !!in_system_header_at (loc)
+	  && !is_location_from_builtin_token (loc))
+	/* The system-ness of this token is different from the one of
+	   the previous token.  Let's emit a line change to mark the
+	   new system-ness before we emit the token.  */
+	{
+	  do_line_change (pfile, token, loc, false);
+	  print.prev_was_system_token = !!in_system_header_at (loc);
+	}
+      cpp_output_token (token, print.outf);
+      line_marker_emitted = false;
+      print.printed = true;
+    }
+
+  /* CPP_COMMENT tokens and raw-string literal tokens can have
+     embedded new-line characters.  Rather than enumerating all the
+     possible token types just check if token uses val.str union
+     member.  */
+  if (cpp_token_val_index (token) == CPP_TOKEN_FLD_STR)
+    account_for_newlines (token->val.str.text, token->val.str.len);
+}
+
 /* Writes out the preprocessed file, handling spacing and paste
    avoidance issues.  */
 static void
@@ -288,10 +422,41 @@ scan_translation_unit (cpp_reader *pfile)
 }
 
 static void
-print_lines_directives_only (int lines, const void *buf, size_t size)
+directives_only_cb (cpp_reader *pfile, CPP_DO_task task, void *data_, ...)
 {
-  print.src_line += lines;
-  fwrite (buf, 1, size, print.outf);
+  va_list args;
+  va_start (args, data_);
+
+  token_streamer *streamer = reinterpret_cast <token_streamer *> (data_);
+  switch (task)
+    {
+    default:
+      gcc_unreachable ();
+
+    case CPP_DO_print:
+      {
+	print.src_line += va_arg (args, unsigned);
+
+	const void *buf = va_arg (args, const void *);
+	size_t size = va_arg (args, size_t);
+	fwrite (buf, 1, size, print.outf);
+      }
+      break;
+
+    case CPP_DO_location:
+      maybe_print_line (va_arg (args, location_t));
+      break;
+
+    case CPP_DO_token:
+      {
+	const cpp_token *token = va_arg (args, const cpp_token *);
+	location_t spelling_loc = va_arg (args, location_t);
+	streamer->stream (pfile, token, spelling_loc);
+      }
+      break;
+    }
+
+  va_end (args);
 }
 
 /* Writes out the preprocessed file, handling spacing and paste
@@ -299,12 +464,8 @@ print_lines_directives_only (int lines, const void *buf, size_t size)
 static void
 scan_translation_unit_directives_only (cpp_reader *pfile)
 {
-  struct _cpp_dir_only_callbacks cb;
-
-  cb.print_lines = print_lines_directives_only;
-  cb.maybe_print_line = maybe_print_line;
-
-  _cpp_preprocess_dir_only (pfile, &cb);
+  token_streamer streamer (pfile);
+  cpp_directive_only_process (pfile, &streamer, directives_only_cb);
 }
 
 /* Adjust print.src_line for newlines embedded in output.  */
@@ -396,7 +557,7 @@ print_line_1 (location_t src_loc, const char *special_flags, FILE *stream)
     putc ('\n', stream);
   print.printed = false;
 
-  if (!flag_no_line_commands)
+  if (src_loc != UNKNOWN_LOCATION && !flag_no_line_commands)
     {
       const char *file_path = LOCATION_FILE (src_loc);
       int sysp;