libctf, link: redo cu-mapping handling

Now a bunch of stuff that doesn't apply to ld or any normal use of
libctf, piled into one commit so that it's easier to ignore.

The cu-mapping machinery associates incoming compilation unit names with
outgoing names of CTF dictionaries that should correspond to them, for
non-gdb CTF consumers that would like to group multiple TUs into a
single child dict if conflicting types are found in it (the existing use
case is one kernel module, one child CTF dict, even if the kernel module
is composed of multiple CUs).

The upcoming deduplicator needs to track not only the mapping from
incoming CU name to outgoing dict name, but the inverse mapping from
outgoing dict name to incoming CU name, so it can work over every CTF
dict we might see in the output and link into it.

So rejig the ctf-link machinery to do that.  Simultaneously (because
they are closely associated and were written at the same time), we add a
new CTF_LINK_EMPTY_CU_MAPPINGS flag to ctf_link, which tells the
ctf_link machinery to create empty child dicts for each outgoing CU
mapping even if no CUs that correspond to it exist in the link.  This is
a bit (OK, quite a lot) of a waste of space, but some existing consumers
require it.  (Nobody else should use it.)

Its value is not consecutive with existing CTF_LINK flag values because
we're about to add more flags that are conceptually closer to the
existing ones than this one is.

include/
	* ctf-api.h (CTF_LINK_EMPTY_CU_MAPPINGS): New.

libctf/
	* ctf-impl.h (ctf_file_t): Improve comments.
	<ctf_link_cu_mapping>: Split into...
	<ctf_link_in_cu_mapping>: ... this...
	<ctf_link_out_cu_mapping>: ... and this.
	* ctf-create.c (ctf_serialize): Adjust.
	* ctf-open.c (ctf_file_close): Likewise.
	* ctf-link.c (ctf_create_per_cu): Look things up in the
	in_cu_mapping instead of the cu_mapping.
	(ctf_link_add_cu_mapping): The deduplicating link will define
	what happens if many FROMs share a TO.
	(ctf_link_add_cu_mapping): Create in_cu_mapping and
	out_cu_mapping. Do not create ctf_link_outputs here any more, or
	create per-CU dicts here: they are already created when needed.
	(ctf_link_one_variable): Log a debug message if we skip a
	variable due to its type being concealed in a CU-mapped link.
	(This is probably too common a case to make into a warning.)
	(ctf_link): Create empty per-CU dicts if requested.

7 files changed, 135 insertions, 32 deletions

diff --git a/include/ChangeLog b/include/ChangeLog
index 6358ec6..60d7b9f 100644
--- a/include/ChangeLog
+++ b/include/ChangeLog
@@ -1,5 +1,9 @@
 2020-07-22  Nick Alcock  <nick.alcock@oracle.com>
 
+	* ctf-api.h (CTF_LINK_EMPTY_CU_MAPPINGS): New.
+
+2020-07-22  Nick Alcock  <nick.alcock@oracle.com>
+
 	* ctf-api.h (ECTF_NEEDSBFD): New.
 	(ECTF_NERR): Adjust.
 	(ctf_link): Rename share_mode arg to flags.
diff --git a/include/ctf-api.h b/include/ctf-api.h
index 7d3e1c8..700a2b1 100644
--- a/include/ctf-api.h
+++ b/include/ctf-api.h
@@ -78,7 +78,7 @@ typedef struct ctf_link_sym
   uint32_t st_value;
 } ctf_link_sym_t;
 
-/* Indication of how to share types when linking.  */
+/* Flags applying to this specific link.  */
 
 /* Share all types that are not in conflict.  The default.  */
 #define CTF_LINK_SHARE_UNCONFLICTED 0x0
@@ -86,6 +86,10 @@ typedef struct ctf_link_sym
 /* Share only types that are used by multiple inputs.  Not implemented yet.  */
 #define CTF_LINK_SHARE_DUPLICATED 0x1
 
+/* Create empty outputs for all registered CU mappings even if no types are
+   emitted into them.  */
+#define CTF_LINK_EMPTY_CU_MAPPINGS 0x4
+
 /* Symbolic names for CTF sections.  */
 
 typedef enum ctf_sect_names
diff --git a/libctf/ChangeLog b/libctf/ChangeLog
index 848bc48..ba6664f 100644
--- a/libctf/ChangeLog
+++ b/libctf/ChangeLog
@@ -1,5 +1,25 @@
 2020-07-22  Nick Alcock  <nick.alcock@oracle.com>
 
+	* ctf-impl.h (ctf_file_t): Improve comments.
+	<ctf_link_cu_mapping>: Split into...
+	<ctf_link_in_cu_mapping>: ... this...
+	<ctf_link_out_cu_mapping>: ... and this.
+	* ctf-create.c (ctf_serialize): Adjust.
+	* ctf-open.c (ctf_file_close): Likewise.
+	* ctf-link.c (ctf_create_per_cu): Look things up in the
+	in_cu_mapping instead of the cu_mapping.
+	(ctf_link_add_cu_mapping): The deduplicating link will define
+	what happens if many FROMs share a TO.
+	(ctf_link_add_cu_mapping): Create in_cu_mapping and
+	out_cu_mapping. Do not create ctf_link_outputs here any more, or
+	create per-CU dicts here: they are already created when needed.
+	(ctf_link_one_variable): Log a debug message if we skip a
+	variable due to its type being concealed in a CU-mapped link.
+	(This is probably too common a case to make into a warning.)
+	(ctf_link): Create empty per-CU dicts if requested.
+
+2020-07-22  Nick Alcock  <nick.alcock@oracle.com>
+
 	* ctf-link.c (ctf_link_write): Close the fd.
 
 2020-07-22  Nick Alcock  <nick.alcock@oracle.com>
diff --git a/libctf/ctf-create.c b/libctf/ctf-create.c
index 85fd060..10c6bbf 100644
--- a/libctf/ctf-create.c
+++ b/libctf/ctf-create.c
@@ -538,7 +538,8 @@ ctf_serialize (ctf_file_t *fp)
   nfp->ctf_errs_warnings = fp->ctf_errs_warnings;
   nfp->ctf_str_prov_offset = fp->ctf_str_prov_offset;
   nfp->ctf_syn_ext_strtab = fp->ctf_syn_ext_strtab;
-  nfp->ctf_link_cu_mapping = fp->ctf_link_cu_mapping;
+  nfp->ctf_link_in_cu_mapping = fp->ctf_link_in_cu_mapping;
+  nfp->ctf_link_out_cu_mapping = fp->ctf_link_out_cu_mapping;
   nfp->ctf_link_type_mapping = fp->ctf_link_type_mapping;
   nfp->ctf_link_memb_name_changer = fp->ctf_link_memb_name_changer;
   nfp->ctf_link_memb_name_changer_arg = fp->ctf_link_memb_name_changer_arg;
@@ -565,7 +566,8 @@ ctf_serialize (ctf_file_t *fp)
   fp->ctf_link_inputs = NULL;
   fp->ctf_link_outputs = NULL;
   fp->ctf_syn_ext_strtab = NULL;
-  fp->ctf_link_cu_mapping = NULL;
+  fp->ctf_link_in_cu_mapping = NULL;
+  fp->ctf_link_out_cu_mapping = NULL;
   fp->ctf_link_type_mapping = NULL;
   fp->ctf_parent_unreffed = 1;
 
diff --git a/libctf/ctf-impl.h b/libctf/ctf-impl.h
index cb7de23..46bceb4 100644
--- a/libctf/ctf-impl.h
+++ b/libctf/ctf-impl.h
@@ -311,12 +311,27 @@ struct ctf_file
   ctf_list_t ctf_errs_warnings;	  /* CTF errors and warnings.  */
   ctf_dynhash_t *ctf_link_inputs; /* Inputs to this link.  */
   ctf_dynhash_t *ctf_link_outputs; /* Additional outputs from this link.  */
-  ctf_dynhash_t *ctf_link_type_mapping; /* Map input types to output types.  */
-  ctf_dynhash_t *ctf_link_cu_mapping;	/* Map CU names to CTF dict names.  */
-  /* Allow the caller to Change the name of link archive members.  */
+
+  /* Map input types to output types: populated in each output dict.
+     Key is a ctf_link_type_key_t: value is a type ID.  Used by
+     nondeduplicating links and ad-hoc ctf_add_type calls only.  */
+  ctf_dynhash_t *ctf_link_type_mapping;
+
+  /* Map input CU names to output CTF dict names: populated in the top-level
+     output dict.
+
+     Key and value are dynamically-allocated strings.  */
+  ctf_dynhash_t *ctf_link_in_cu_mapping;
+
+  /* Map output CTF dict names to input CU names: populated in the top-level
+     output dict.  A hash of string to hash (set) of strings.  Key and
+     individual value members are shared with ctf_link_in_cu_mapping.  */
+  ctf_dynhash_t *ctf_link_out_cu_mapping;
+
   /* CTF linker flags.  */
   int ctf_link_flags;
 
+  /* Allow the caller to change the name of link archive members.  */
   ctf_link_memb_name_changer_f *ctf_link_memb_name_changer;
   void *ctf_link_memb_name_changer_arg; /* Argument for it.  */
   ctf_dynhash_t *ctf_add_processing; /* Types ctf_add_type is working on now.  */
diff --git a/libctf/ctf-link.c b/libctf/ctf-link.c
index fa15c9b..3c96604 100644
--- a/libctf/ctf-link.c
+++ b/libctf/ctf-link.c
@@ -296,10 +296,12 @@ ctf_create_per_cu (ctf_file_t *fp, const char *filename, const char *cuname)
      dictionary name.  We prefer the filename because this is easier for likely
      callers to determine.  */
 
-  if (fp->ctf_link_cu_mapping)
+  if (fp->ctf_link_in_cu_mapping)
     {
-      if (((ctf_name = ctf_dynhash_lookup (fp->ctf_link_cu_mapping, filename)) == NULL) &&
-	  ((ctf_name = ctf_dynhash_lookup (fp->ctf_link_cu_mapping, cuname)) == NULL))
+      if (((ctf_name = ctf_dynhash_lookup (fp->ctf_link_in_cu_mapping,
+					   filename)) == NULL) &&
+	  ((ctf_name = ctf_dynhash_lookup (fp->ctf_link_in_cu_mapping,
+					   cuname)) == NULL))
 	ctf_name = filename;
     }
 
@@ -339,10 +341,7 @@ ctf_create_per_cu (ctf_file_t *fp, const char *filename, const char *cuname)
 
 /* Add a mapping directing that the CU named FROM should have its
    conflicting/non-duplicate types (depending on link mode) go into a container
-   named TO.  Many FROMs can share a TO: in this case, the effect on conflicting
-   types is not yet defined (but in time an auto-renaming algorithm will be
-   added: ugly, but there is really no right thing one can do in this
-   situation).
+   named TO.  Many FROMs can share a TO.
 
    We forcibly add a container named TO in every case, even though it may well
    wind up empty, because clients that use this facility usually expect to find
@@ -352,34 +351,63 @@ int
 ctf_link_add_cu_mapping (ctf_file_t *fp, const char *from, const char *to)
 {
   int err;
-  char *f, *t;
+  char *f = NULL, *t = NULL;
+  ctf_dynhash_t *one_out;
+
+  if (fp->ctf_link_in_cu_mapping == NULL)
+    fp->ctf_link_in_cu_mapping = ctf_dynhash_create (ctf_hash_string,
+						     ctf_hash_eq_string, free,
+						     free);
+  if (fp->ctf_link_in_cu_mapping == NULL)
+    goto oom;
 
-  if (fp->ctf_link_cu_mapping == NULL)
-    fp->ctf_link_cu_mapping = ctf_dynhash_create (ctf_hash_string,
-						  ctf_hash_eq_string, free,
-						  free);
-  if (fp->ctf_link_cu_mapping == NULL)
-    return ctf_set_errno (fp, ENOMEM);
+  if (fp->ctf_link_out_cu_mapping == NULL)
+    fp->ctf_link_out_cu_mapping = ctf_dynhash_create (ctf_hash_string,
+						      ctf_hash_eq_string, free,
+						      (ctf_hash_free_fun)
+						      ctf_dynhash_destroy);
+  if (fp->ctf_link_out_cu_mapping == NULL)
+    goto oom;
 
-  if (fp->ctf_link_outputs == NULL)
-    fp->ctf_link_outputs = ctf_dynhash_create (ctf_hash_string,
-					       ctf_hash_eq_string, free,
-					       (ctf_hash_free_fun)
-					       ctf_file_close);
+  f = strdup (from);
+  t = strdup (to);
+  if (!f || !t)
+    goto oom;
 
-  if (fp->ctf_link_outputs == NULL)
-    return ctf_set_errno (fp, ENOMEM);
+  /* Track both in a list from FROM to TO and in a list from TO to a list of
+     FROM.  The former is used to create TUs with the mapped-to name at need:
+     the latter is used in deduplicating links to pull in all input CUs
+     corresponding to a single output CU.  */
+
+  if ((err = ctf_dynhash_insert (fp->ctf_link_in_cu_mapping, f, t)) < 0)
+    {
+      ctf_set_errno (fp, err);
+      goto oom_noerrno;
+    }
 
+  /* f and t are now owned by the in_cu_mapping: reallocate them.  */
   f = strdup (from);
   t = strdup (to);
   if (!f || !t)
     goto oom;
 
-  if (ctf_create_per_cu (fp, t, t) == NULL)
-    goto oom_noerrno;				/* Errno is set for us.  */
+  if ((one_out = ctf_dynhash_lookup (fp->ctf_link_out_cu_mapping, t)) == NULL)
+    {
+      if ((one_out = ctf_dynhash_create (ctf_hash_string, ctf_hash_eq_string,
+					 free, NULL)) == NULL)
+	goto oom;
+      if ((err = ctf_dynhash_insert (fp->ctf_link_out_cu_mapping,
+				     t, one_out)) < 0)
+	{
+	  ctf_dynhash_destroy (one_out);
+	  ctf_set_errno (fp, err);
+	  goto oom_noerrno;
+	}
+    }
+  else
+    free (t);
 
-  err = ctf_dynhash_insert (fp->ctf_link_cu_mapping, f, t);
-  if (err)
+  if (ctf_dynhash_insert (one_out, f, NULL) < 0)
     {
       ctf_set_errno (fp, err);
       goto oom_noerrno;
@@ -777,6 +805,8 @@ int
 ctf_link (ctf_file_t *fp, int flags)
 {
   ctf_link_in_member_cb_arg_t arg;
+  ctf_next_t *i = NULL;
+  int err;
 
   memset (&arg, 0, sizeof (struct ctf_link_in_member_cb_arg));
   arg.out_fp = fp;
@@ -794,6 +824,33 @@ ctf_link (ctf_file_t *fp, int flags)
   if (fp->ctf_link_outputs == NULL)
     return ctf_set_errno (fp, ENOMEM);
 
+  /* Create empty CUs if requested.  We do not currently claim that multiple
+     links in succession with CTF_LINK_EMPTY_CU_MAPPINGS set in some calls and
+     not set in others will do anything especially sensible.  */
+
+  if (fp->ctf_link_out_cu_mapping && (flags & CTF_LINK_EMPTY_CU_MAPPINGS))
+    {
+      void *v;
+
+      while ((err = ctf_dynhash_next (fp->ctf_link_out_cu_mapping, &i, &v,
+				      NULL)) == 0)
+	{
+	  const char *to = (const char *) v;
+	  if (ctf_create_per_cu (fp, to, to) == NULL)
+	    {
+	      ctf_next_destroy (i);
+	      return -1;			/* Errno is set for us.  */
+	    }
+	}
+      if (err != ECTF_NEXT_END)
+	{
+	  ctf_err_warn (fp, 1, "Iteration error creating empty CUs: %s",
+			ctf_errmsg (err));
+	  ctf_set_errno (fp, err);
+	  return -1;
+	}
+    }
+
   ctf_dynhash_iter (fp->ctf_link_inputs, ctf_link_one_input_archive,
 		    &arg);
 
diff --git a/libctf/ctf-open.c b/libctf/ctf-open.c
index 87bff2f..285e0e0 100644
--- a/libctf/ctf-open.c
+++ b/libctf/ctf-open.c
@@ -1716,7 +1716,8 @@ ctf_file_close (ctf_file_t *fp)
   ctf_dynhash_destroy (fp->ctf_link_inputs);
   ctf_dynhash_destroy (fp->ctf_link_outputs);
   ctf_dynhash_destroy (fp->ctf_link_type_mapping);
-  ctf_dynhash_destroy (fp->ctf_link_cu_mapping);
+  ctf_dynhash_destroy (fp->ctf_link_in_cu_mapping);
+  ctf_dynhash_destroy (fp->ctf_link_out_cu_mapping);
   ctf_dynhash_destroy (fp->ctf_add_processing);
 
   for (err = ctf_list_next (&fp->ctf_errs_warnings); err != NULL; err = nerr)