p1689r5: initial support

This patch implements support for [P1689R5][] to communicate to a build
system the C++20 module dependencies to build systems so that they may
build `.gcm` files in the proper order.

Support is communicated through the following three new flags:

- `-fdeps-format=` specifies the format for the output. Currently named
  `p1689r5`.

- `-fdeps-file=` specifies the path to the file to write the format to.

- `-fdeps-target=` specifies the `.o` that will be written for the TU
  that is scanned. This is required so that the build system can
  correlate the dependency output with the actual compilation that will
  occur.

CMake supports this format as of 17 Jun 2022 (to be part of 3.25.0)
using an experimental feature selection (to allow for future usage
evolution without committing to how it works today). While it remains
experimental, docs may be found in CMake's documentation for
experimental features.

Future work may include using this format for Fortran module
dependencies as well, however this is still pending work.

[P1689R5]: https://isocpp.org/files/papers/P1689R5.html
[cmake-experimental]: https://gitlab.kitware.com/cmake/cmake/-/blob/master/Help/dev/experimental.rst

TODO:

- header-unit information fields

Header units (including the standard library headers) are 100%
unsupported right now because the `-E` mechanism wants to import their
BMIs. A new mode (i.e., something more workable than existing `-E`
behavior) that mocks up header units as if they were imported purely
from their path and content would be required.

- non-utf8 paths

The current standard says that paths that are not unambiguously
represented using UTF-8 are not supported (because these cases are rare
and the extra complication is not worth it at this time). Future
versions of the format might have ways of encoding non-UTF-8 paths. For
now, this patch just doesn't support non-UTF-8 paths (ignoring the
"unambiguously representable in UTF-8" case).

- figure out why junk gets placed at the end of the file

Sometimes it seems like the file gets a lot of `NUL` bytes appended to
it. It happens rarely and seems to be the result of some
`ftruncate`-style call which results in extra padding in the contents.
Noting it here as an observation at least.

libcpp/

	* include/cpplib.h: Add cpp_fdeps_format enum.
	(cpp_options): Add fdeps_format field
	(cpp_finish): Add structured dependency fdeps_stream parameter.
	* include/mkdeps.h (deps_add_module_target): Add flag for
	whether a module is exported or not.
	(fdeps_add_target): Add function.
	(deps_write_p1689r5): Add function.
	* init.cc (cpp_finish): Add new preprocessor parameter used for C++
	module tracking.
	* mkdeps.cc (mkdeps): Implement P1689R5 output.

gcc/

	* doc/invoke.texi: Document -fdeps-format=, -fdeps-file=, and
	-fdeps-target= flags.
	* gcc.cc: add defaults for -fdeps-target= and -fdeps-file= when
	only -fdeps-format= is specified.
	* json.h: Add a TODO item to refactor out to share with
	`libcpp/mkdeps.cc`.

gcc/c-family/

	* c-opts.cc (c_common_handle_option): Add fdeps_file variable and
	-fdeps-format=, -fdeps-file=, and -fdeps-target= parsing.
	* c.opt: Add -fdeps-format=, -fdeps-file=, and -fdeps-target=
	flags.

gcc/cp/

	* module.cc (preprocessed_module): Pass whether the module is
	exported to dependency tracking.

gcc/testsuite/

	* g++.dg/modules/depflags-f-MD.C: New test.
	* g++.dg/modules/depflags-f.C: New test.
	* g++.dg/modules/depflags-fi.C: New test.
	* g++.dg/modules/depflags-fj-MD.C: New test.
	* g++.dg/modules/depflags-fj.C: New test.
	* g++.dg/modules/depflags-fjo-MD.C: New test.
	* g++.dg/modules/depflags-fjo.C: New test.
	* g++.dg/modules/depflags-fo-MD.C: New test.
	* g++.dg/modules/depflags-fo.C: New test.
	* g++.dg/modules/depflags-j-MD.C: New test.
	* g++.dg/modules/depflags-j.C: New test.
	* g++.dg/modules/depflags-jo-MD.C: New test.
	* g++.dg/modules/depflags-jo.C: New test.
	* g++.dg/modules/depflags-o-MD.C: New test.
	* g++.dg/modules/depflags-o.C: New test.
	* g++.dg/modules/p1689-1.C: New test.
	* g++.dg/modules/p1689-1.exp.ddi: New test expectation.
	* g++.dg/modules/p1689-2.C: New test.
	* g++.dg/modules/p1689-2.exp.ddi: New test expectation.
	* g++.dg/modules/p1689-3.C: New test.
	* g++.dg/modules/p1689-3.exp.ddi: New test expectation.
	* g++.dg/modules/p1689-4.C: New test.
	* g++.dg/modules/p1689-4.exp.ddi: New test expectation.
	* g++.dg/modules/p1689-5.C: New test.
	* g++.dg/modules/p1689-5.exp.ddi: New test expectation.
	* g++.dg/modules/modules.exp: Load new P1689 library routines.
	* g++.dg/modules/test-p1689.py: New tool for validating P1689 output.
	* lib/modules.exp: Support for validating P1689 outputs.

Signed-off-by: Ben Boeckel <ben.boeckel@kitware.com>
Reviewed-by: Jason Merrill <jason@redhat.com>

4 files changed, 186 insertions, 11 deletions

diff --git a/libcpp/include/cpplib.h b/libcpp/include/cpplib.h
index fcdaf08..c0af82c 100644
--- a/libcpp/include/cpplib.h
+++ b/libcpp/include/cpplib.h
@@ -302,6 +302,9 @@ typedef CPPCHAR_SIGNED_T cppchar_signed_t;
 /* Style of header dependencies to generate.  */
 enum cpp_deps_style { DEPS_NONE = 0, DEPS_USER, DEPS_SYSTEM };
 
+/* Structured format of module dependencies to generate.  */
+enum cpp_fdeps_format { FDEPS_FMT_NONE = 0, FDEPS_FMT_P1689R5 };
+
 /* The possible normalization levels, from most restrictive to least.  */
 enum cpp_normalize_level {
   /* In NFKC.  */
@@ -589,6 +592,9 @@ struct cpp_options
     /* Style of header dependencies to generate.  */
     enum cpp_deps_style style;
 
+    /* Structured format of module dependencies to generate.  */
+    enum cpp_fdeps_format fdeps_format;
+
     /* Assume missing files are generated files.  */
     bool missing_files;
 
@@ -1112,9 +1118,9 @@ extern void cpp_post_options (cpp_reader *);
 extern void cpp_init_iconv (cpp_reader *);
 
 /* Call this to finish preprocessing.  If you requested dependency
-   generation, pass an open stream to write the information to,
-   otherwise NULL.  It is your responsibility to close the stream.  */
-extern void cpp_finish (cpp_reader *, FILE *deps_stream);
+   generation, pass open stream(s) to write the information to,
+   otherwise NULL.  It is your responsibility to close the stream(s).  */
+extern void cpp_finish (cpp_reader *, FILE *deps_stream, FILE *fdeps_stream = NULL);
 
 /* Call this to release the handle at the end of preprocessing.  Any
    use of the handle after this function returns is invalid.  */
diff --git a/libcpp/include/mkdeps.h b/libcpp/include/mkdeps.h
index 920e279..7025e63 100644
--- a/libcpp/include/mkdeps.h
+++ b/libcpp/include/mkdeps.h
@@ -53,11 +53,15 @@ extern void deps_add_default_target (class mkdeps *, const char *);
 
 /* Adds a module target.  The module name and cmi name are copied.  */
 extern void deps_add_module_target (struct mkdeps *, const char *module,
-				    const char *cmi, bool is_header);
+				    const char *cmi, bool is_header,
+				    bool is_exported);
 
 /* Adds a module dependency.  The module name is copied.  */
 extern void deps_add_module_dep (struct mkdeps *, const char *module);
 
+/* Add a structured dependency target.  */
+extern void fdeps_add_target (struct mkdeps *, const char *, bool);
+
 /* Add a dependency (appears on the right side of the colon) to the
    deps list.  Dependencies will be printed in the order that they
    were entered with this function.  By convention, the first
@@ -68,6 +72,9 @@ extern void deps_add_dep (class mkdeps *, const char *);
    is the number of columns to word-wrap at (0 means don't wrap).  */
 extern void deps_write (const cpp_reader *, FILE *, unsigned int);
 
+/* Write out a deps buffer to a specified file in P1689R5 format.  */
+extern void deps_write_p1689r5 (const struct mkdeps *, FILE *);
+
 /* Write out a deps buffer to a file, in a form that can be read back
    with deps_restore.  Returns nonzero on error, in which case the
    error number will be in errno.  */
diff --git a/libcpp/init.cc b/libcpp/init.cc
index 693feaa..9a20f8d 100644
--- a/libcpp/init.cc
+++ b/libcpp/init.cc
@@ -860,7 +860,7 @@ read_original_directory (cpp_reader *pfile)
    Maybe it should also reset state, such that you could call
    cpp_start_read with a new filename to restart processing.  */
 void
-cpp_finish (cpp_reader *pfile, FILE *deps_stream)
+cpp_finish (struct cpp_reader *pfile, FILE *deps_stream, FILE *fdeps_stream)
 {
   /* Warn about unused macros before popping the final buffer.  */
   if (CPP_OPTION (pfile, warn_unused_macros))
@@ -874,8 +874,15 @@ cpp_finish (cpp_reader *pfile, FILE *deps_stream)
   while (pfile->buffer)
     _cpp_pop_buffer (pfile);
 
-  if (deps_stream)
-    deps_write (pfile, deps_stream, 72);
+  cpp_fdeps_format fdeps_format = CPP_OPTION (pfile, deps.fdeps_format);
+  if (fdeps_format == FDEPS_FMT_P1689R5 && fdeps_stream)
+    deps_write_p1689r5 (pfile->deps, fdeps_stream);
+
+  if (CPP_OPTION (pfile, deps.style) != DEPS_NONE
+      && deps_stream)
+    {
+      deps_write (pfile, deps_stream, 72);
+    }
 
   /* Report on headers that could use multiple include guards.  */
   if (CPP_OPTION (pfile, print_include_names))
diff --git a/libcpp/mkdeps.cc b/libcpp/mkdeps.cc
index 8f9585c..a3e2c78 100644
--- a/libcpp/mkdeps.cc
+++ b/libcpp/mkdeps.cc
@@ -81,7 +81,8 @@ public:
   };
 
   mkdeps ()
-    : module_name (NULL), cmi_name (NULL), is_header_unit (false), quote_lwm (0)
+    : primary_output (NULL), module_name (NULL), cmi_name (NULL)
+    , is_header_unit (false), is_exported (false), quote_lwm (0)
   {
   }
   ~mkdeps ()
@@ -90,6 +91,9 @@ public:
 
     for (i = targets.size (); i--;)
       free (const_cast <char *> (targets[i]));
+    free (const_cast <char *> (primary_output));
+    for (i = fdeps_targets.size (); i--;)
+      free (const_cast <char *> (fdeps_targets[i]));
     for (i = deps.size (); i--;)
       free (const_cast <char *> (deps[i]));
     for (i = vpath.size (); i--;)
@@ -103,6 +107,8 @@ public:
 public:
   vec<const char *> targets;
   vec<const char *> deps;
+  const char * primary_output;
+  vec<const char *> fdeps_targets;
   vec<velt> vpath;
   vec<const char *> modules;
 
@@ -110,6 +116,7 @@ public:
   const char *module_name;
   const char *cmi_name;
   bool is_header_unit;
+  bool is_exported;
   unsigned short quote_lwm;
 };
 
@@ -288,6 +295,26 @@ deps_add_default_target (class mkdeps *d, const char *tgt)
     }
 }
 
+/* Adds a target O.  We make a copy, so it need not be a permanent
+   string.
+
+   This is the target associated with the rule that (in a C++ modules build)
+   compiles the source that is being scanned for dynamic dependencies.  It is
+   used to associate the structured dependency information with that rule as
+   needed.  */
+void
+fdeps_add_target (struct mkdeps *d, const char *o, bool is_primary)
+{
+  o = apply_vpath (d, o);
+  if (is_primary)
+  {
+    if (d->primary_output)
+      d->fdeps_targets.push (d->primary_output);
+    d->primary_output = xstrdup (o);
+  } else
+    d->fdeps_targets.push (xstrdup (o));
+}
+
 void
 deps_add_dep (class mkdeps *d, const char *t)
 {
@@ -325,12 +352,13 @@ deps_add_vpath (class mkdeps *d, const char *vpath)
 
 void
 deps_add_module_target (struct mkdeps *d, const char *m,
-			const char *cmi, bool is_header_unit)
+			const char *cmi, bool is_header_unit, bool is_exported)
 {
   gcc_assert (!d->module_name);
   
   d->module_name = xstrdup (m);
   d->is_header_unit = is_header_unit;
+  d->is_exported = is_exported;
   d->cmi_name = xstrdup (cmi);
 }
 
@@ -395,10 +423,16 @@ make_write (const cpp_reader *pfile, FILE *fp, unsigned int colmax)
   if (colmax && colmax < 34)
     colmax = 34;
 
+  /* Write out C++ modules information if no other `-fdeps-format=`
+     option is given. */
+  cpp_fdeps_format fdeps_format = CPP_OPTION (pfile, deps.fdeps_format);
+  bool write_make_modules_deps = (fdeps_format == FDEPS_FMT_NONE
+				  && CPP_OPTION (pfile, deps.modules));
+
   if (d->deps.size ())
     {
       column = make_write_vec (d->targets, fp, 0, colmax, d->quote_lwm);
-      if (CPP_OPTION (pfile, deps.modules) && d->cmi_name)
+      if (write_make_modules_deps && d->cmi_name)
 	column = make_write_name (d->cmi_name, fp, column, colmax);
       fputs (":", fp);
       column++;
@@ -409,7 +443,7 @@ make_write (const cpp_reader *pfile, FILE *fp, unsigned int colmax)
 	  fprintf (fp, "%s:\n", munge (d->deps[i]));
     }
 
-  if (!CPP_OPTION (pfile, deps.modules))
+  if (!write_make_modules_deps)
     return;
 
   if (d->modules.size ())
@@ -473,6 +507,127 @@ deps_write (const cpp_reader *pfile, FILE *fp, unsigned int colmax)
   make_write (pfile, fp, colmax);
 }
 
+/* Write out a a filepath for P1689R5 output.  */
+
+static void
+p1689r5_write_filepath (const char *name, FILE *fp)
+{
+  if (cpp_valid_utf8_p (name, strlen (name)))
+    {
+      fputc ('"', fp);
+      for (const char* c = name; *c; c++)
+	{
+	  // Escape control characters.
+	  if (ISCNTRL (*c))
+	    fprintf (fp, "\\u%04x", *c);
+	  // JSON escape characters.
+	  else if (*c == '"' || *c == '\\')
+	    {
+	      fputc ('\\', fp);
+	      fputc (*c, fp);
+	    }
+	  // Everything else.
+	  else
+	    fputc (*c, fp);
+	}
+      fputc ('"', fp);
+    }
+  else
+    {
+      // TODO: print an error
+    }
+}
+
+/* Write a JSON array from a `vec` for P1689R5 output.
+
+   In P1689R5, all array values are filepaths.  */
+
+static void
+p1689r5_write_vec (const mkdeps::vec<const char *> &vec, FILE *fp)
+{
+  for (unsigned ix = 0; ix != vec.size (); ix++)
+    {
+      p1689r5_write_filepath (vec[ix], fp);
+      if (ix < vec.size () - 1)
+	fputc (',', fp);
+      fputc ('\n', fp);
+    }
+}
+
+/* Write out the P1689R5 format using the module dependency tracking
+   information gathered while scanning and/or compiling.
+
+   Ideally this (and the above `p1689r5_` functions) would use `gcc/json.h`,
+   but since this is `libcpp`, we cannot use `gcc/` code.
+
+  TODO: move `json.h` to libiberty.  */
+
+void
+deps_write_p1689r5 (const struct mkdeps *d, FILE *fp)
+{
+  fputs ("{\n", fp);
+
+  fputs ("\"rules\": [\n", fp);
+  fputs ("{\n", fp);
+
+  if (d->primary_output)
+    {
+      fputs ("\"primary-output\": ", fp);
+      p1689r5_write_filepath (d->primary_output, fp);
+      fputs (",\n", fp);
+    }
+
+  if (d->fdeps_targets.size ())
+    {
+      fputs ("\"outputs\": [\n", fp);
+      p1689r5_write_vec (d->fdeps_targets, fp);
+      fputs ("],\n", fp);
+    }
+
+  if (d->module_name)
+    {
+      fputs ("\"provides\": [\n", fp);
+      fputs ("{\n", fp);
+
+      fputs ("\"logical-name\": ", fp);
+      p1689r5_write_filepath (d->module_name, fp);
+      fputs (",\n", fp);
+
+      fprintf (fp, "\"is-interface\": %s\n", d->is_exported ? "true" : "false");
+
+      // TODO: header-unit information
+
+      fputs ("}\n", fp);
+      fputs ("],\n", fp);
+    }
+
+  fputs ("\"requires\": [\n", fp);
+  for (size_t i = 0; i < d->modules.size (); i++)
+    {
+      if (i != 0)
+	fputs (",\n", fp);
+      fputs ("{\n", fp);
+
+      fputs ("\"logical-name\": ", fp);
+      p1689r5_write_filepath (d->modules[i], fp);
+      fputs ("\n", fp);
+
+      // TODO: header-unit information
+
+      fputs ("}\n", fp);
+    }
+  fputs ("]\n", fp);
+
+  fputs ("}\n", fp);
+
+  fputs ("],\n", fp);
+
+  fputs ("\"version\": 0,\n", fp);
+  fputs ("\"revision\": 0\n", fp);
+
+  fputs ("}\n", fp);
+}
+
 /* Write out a deps buffer to a file, in a form that can be read back
    with deps_restore.  Returns nonzero on error, in which case the
    error number will be in errno.  */