libctf, include: CTF-archive-wide symbol lookup

CTF archives may contain multiple dicts, each of which contain many
types and possibly a bunch of symtypetab entries relating to those
types: each symtypetab entry is going to appear in exactly one dict,
with the corresponding entries in the other dicts empty (either pads, or
indexed symtypetabs that do not mention that symbol).  But users of
libctf usually want to get back the type associated with a symbol
without having to dig around to find out which dict that type might be
in.

This adds machinery to do that -- and since you probably want to do it
repeatedly, it adds internal caching to the ctf-archive machinery so
that iteration over archives via ctf_archive_next and repeated symbol
lookups do not have to repeatedly reopen the archive.  (Iteration using
ctf_archive_iter will gain caching soon.)

Two new API functions:

ctf_dict_t *
ctf_arc_lookup_symbol (ctf_archive_t *arc, unsigned long symidx,
		       ctf_id_t *typep, int *errp);

This looks up the symbol with index SYMIDX in the archive ARC, returning
the dictionary in which it resides and optionally the type index as
well.  Errors are returned in ERRP.  The dict should be
ctf_dict_close()d when done, but is also cached inside the ctf_archive
so that the open cost is only paid once.  The result of the symbol
lookup is also cached internally, so repeated lookups of the same symbol
are nearly free.

void ctf_arc_flush_caches (ctf_archive_t *arc);

Flush all the caches. Done at close time, but also available as an API
function if users want to do it by hand.

include/ChangeLog
2020-11-20  Nick Alcock  <nick.alcock@oracle.com>

	* ctf-api.h (ctf_arc_lookup_symbol): New.
	(ctf_arc_flush_caches): Likewise.
	* ctf.h: Document new auto-ctf_import behaviour.

libctf/ChangeLog
2020-11-20  Nick Alcock  <nick.alcock@oracle.com>

	* ctf-impl.h (struct ctf_archive_internal) <ctfi_dicts>: New, dicts
	the archive machinery has opened and cached.
	<ctfi_symdicts>: New, cache of dicts containing symbols looked up.
	<ctfi_syms>: New, cache of types of symbols looked up.
	* ctf-archive.c (ctf_arc_close): Free them on close.
	(enosym): New, flag entry for 'symbol not present'.
	(ctf_arc_import_parent): New, automatically import the parent from
	".ctf" if this is a child in an archive and ".ctf" is present.
	(ctf_dict_open_sections): Use it.
	(ctf_archive_iter_internal): Likewise.
	(ctf_cached_dict_close): New, thunk around ctf_dict_close.
	(ctf_dict_open_cached): New, open and cache a dict.
	(ctf_arc_flush_caches): New, flush the caches.
	(ctf_arc_lookup_symbol): New, look up a symbol in (all members of)
	an archive, and cache the lookup.
	(ctf_archive_iter): Note the new caching behaviour.
	(ctf_archive_next): Use ctf_dict_open_cached.
	* libctf.ver: Add ctf_arc_lookup_symbol and ctf_arc_flush_caches.

7 files changed, 275 insertions, 18 deletions

diff --git a/include/ChangeLog b/include/ChangeLog
index ad4c8a6..1d4eac9 100644
--- a/include/ChangeLog
+++ b/include/ChangeLog
@@ -1,5 +1,11 @@
 2020-11-20  Nick Alcock  <nick.alcock@oracle.com>
 
+	* ctf-api.h (ctf_arc_lookup_symbol): New.
+	(ctf_arc_flush_caches): Likewise.
+	* ctf.h: Document new auto-ctf_import behaviour.
+
+2020-11-20  Nick Alcock  <nick.alcock@oracle.com>
+
 	* ctf-api.h (ctf_symbol_next): New.
 	(ctf_add_objt_sym): Likewise.
 	(ctf_add_func_sym): Likewise.
diff --git a/include/ctf-api.h b/include/ctf-api.h
index 6dd37b9..f0c00c0 100644
--- a/include/ctf-api.h
+++ b/include/ctf-api.h
@@ -109,7 +109,9 @@ typedef enum ctf_sect_names
    CTF_SECT_HEADER,
    CTF_SECT_LABEL,
    CTF_SECT_OBJT,
+   CTF_SECT_OBJTIDX = CTF_SECT_OBJT,
    CTF_SECT_FUNC,
+   CTF_SECT_FUNCIDX = CTF_SECT_FUNC,
    CTF_SECT_VAR,
    CTF_SECT_TYPE,
    CTF_SECT_STR
@@ -312,6 +314,10 @@ extern ctf_archive_t *ctf_arc_bufopen (const ctf_sect_t *,
 				       const ctf_sect_t *,
 				       int *);
 extern void ctf_arc_close (ctf_archive_t *);
+extern ctf_dict_t *ctf_arc_lookup_symbol (ctf_archive_t *,
+					  unsigned long symidx,
+					  ctf_id_t *, int *errp);
+extern void ctf_arc_flush_caches (ctf_archive_t *);
 extern ctf_dict_t *ctf_dict_open (const ctf_archive_t *,
 				  const char *, int *);
 extern ctf_dict_t *ctf_dict_open_sections (const ctf_archive_t *,
diff --git a/include/ctf.h b/include/ctf.h
index c7a1e43..0800287 100644
--- a/include/ctf.h
+++ b/include/ctf.h
@@ -116,9 +116,12 @@ extern "C"
    and libctf library are responsible for connecting the appropriate objects
    together so that the full set of types can be explored and manipulated.
 
-   This connection is done purely using the ctf_import() function.  There is no
-   notation anywhere in the child CTF file indicating which parent it is
-   connected to: it is the debugger's responsibility to track this.  */
+   This connection is done purely using the ctf_import() function.  The
+   ctf_archive machinery (and thus ctf_open et al) automatically imports archive
+   members named ".ctf" into child dicts if available in the same archive, to
+   match the relationship set up by the linker, but callers can call ctf_import
+   themselves as well if need be, if they know a different relationship is in
+   force.  */
 
 #define CTF_MAX_TYPE	0xfffffffe	/* Max type identifier value.  */
 #define CTF_MAX_PTYPE	0x7fffffff	/* Max parent type identifier value.  */
diff --git a/libctf/ChangeLog b/libctf/ChangeLog
index e571404..deca027 100644
--- a/libctf/ChangeLog
+++ b/libctf/ChangeLog
@@ -1,5 +1,26 @@
 2020-11-20  Nick Alcock  <nick.alcock@oracle.com>
 
+	* ctf-impl.h (struct ctf_archive_internal) <ctfi_dicts>: New, dicts
+	the archive machinery has opened and cached.
+	<ctfi_symdicts>: New, cache of dicts containing symbols looked up.
+	<ctfi_syms>: New, cache of types of symbols looked up.
+	* ctf-archive.c (ctf_arc_close): Free them on close.
+	(enosym): New, flag entry for 'symbol not present'.
+	(ctf_arc_import_parent): New, automatically import the parent from
+	".ctf" if this is a child in an archive and ".ctf" is present.
+	(ctf_dict_open_sections): Use it.
+	(ctf_archive_iter_internal): Likewise.
+	(ctf_cached_dict_close): New, thunk around ctf_dict_close.
+	(ctf_dict_open_cached): New, open and cache a dict.
+	(ctf_arc_flush_caches): New, flush the caches.
+	(ctf_arc_lookup_symbol): New, look up a symbol in (all members of)
+	an archive, and cache the lookup.
+	(ctf_archive_iter): Note the new caching behaviour.
+	(ctf_archive_next): Use ctf_dict_open_cached.
+	* libctf.ver: Add ctf_arc_lookup_symbol and ctf_arc_flush_caches.
+
+2020-11-20  Nick Alcock  <nick.alcock@oracle.com>
+
 	* ctf-dedup.c (ctf_dedup_rhash_type): Null out the names of nameless
 	type kinds, just in case the input has named them.
 
diff --git a/libctf/ctf-archive.c b/libctf/ctf-archive.c
index 25c30f6..dc312d3 100644
--- a/libctf/ctf-archive.c
+++ b/libctf/ctf-archive.c
@@ -43,6 +43,11 @@ static void *arc_mmap_file (int fd, size_t size);
 static int arc_mmap_writeout (int fd, void *header, size_t headersz,
 			      const char **errmsg);
 static int arc_mmap_unmap (void *header, size_t headersz, const char **errmsg);
+static void ctf_arc_import_parent (const ctf_archive_t *arc, ctf_dict_t *fp);
+
+/* Flag to indicate "symbol not present" in
+   ctf_archive_internal.ctfi_symdicts.  Never initialized.  */
+static ctf_dict_t enosym;
 
 /* Write out a CTF archive to the start of the file referenced by the passed-in
    fd.  The entries in CTF_DICTS are referenced by name: the names are passed in
@@ -512,6 +517,9 @@ ctf_arc_close (ctf_archive_t *arc)
     }
   else
     ctf_dict_close (arc->ctfi_dict);
+  free (arc->ctfi_syms);
+  free (arc->ctfi_symdicts);
+  ctf_dynhash_destroy (arc->ctfi_dicts);
   if (arc->ctfi_free_symsect)
     free ((void *) arc->ctfi_symsect.cts_data);
   if (arc->ctfi_free_strsect)
@@ -578,7 +586,10 @@ ctf_dict_open_sections (const ctf_archive_t *arc,
       ret = ctf_dict_open_internal (arc->ctfi_archive, symsect, strsect,
 				    name, errp);
       if (ret)
-	ret->ctf_archive = (ctf_archive_t *) arc;
+	{
+	  ret->ctf_archive = (ctf_archive_t *) arc;
+	  ctf_arc_import_parent (arc, ret);
+	}
       return ret;
     }
 
@@ -613,6 +624,67 @@ ctf_dict_open (const ctf_archive_t *arc, const char *name, int *errp)
   return ctf_dict_open_sections (arc, symsect, strsect, name, errp);
 }
 
+static void
+ctf_cached_dict_close (void *fp)
+{
+  ctf_dict_close ((ctf_dict_t *) fp);
+}
+
+/* Return the ctf_dict_t with the given name and cache it in the
+   archive's ctfi_dicts.  */
+static ctf_dict_t *
+ctf_dict_open_cached (ctf_archive_t *arc, const char *name, int *errp)
+{
+  ctf_dict_t *fp;
+  char *dupname;
+
+  /* Just return from the cache if possible.  */
+  if (arc->ctfi_dicts
+      && ((fp = ctf_dynhash_lookup (arc->ctfi_dicts, name)) != NULL))
+    {
+      fp->ctf_refcnt++;
+      return fp;
+    }
+
+  /* Not yet cached: open it.  */
+  fp = ctf_dict_open (arc, name, errp);
+  dupname = strdup (name);
+
+  if (!fp || !dupname)
+    goto oom;
+
+  if (arc->ctfi_dicts == NULL)
+    if ((arc->ctfi_dicts
+	 = ctf_dynhash_create (ctf_hash_string, ctf_hash_eq_string,
+			       free, ctf_cached_dict_close)) == NULL)
+      goto oom;
+
+  if (ctf_dynhash_insert (arc->ctfi_dicts, dupname, fp) < 0)
+    goto oom;
+  fp->ctf_refcnt++;
+
+  return fp;
+
+ oom:
+  ctf_dict_close (fp);
+  free (dupname);
+  if (errp)
+    *errp = ENOMEM;
+  return NULL;
+}
+
+/* Flush any caches the CTF archive may have open.  */
+void
+ctf_arc_flush_caches (ctf_archive_t *wrapper)
+{
+  free (wrapper->ctfi_symdicts);
+  free (wrapper->ctfi_syms);
+  ctf_dynhash_destroy (wrapper->ctfi_dicts);
+  wrapper->ctfi_symdicts = NULL;
+  wrapper->ctfi_syms = NULL;
+  wrapper->ctfi_dicts = NULL;
+}
+
 /* Return the ctf_dict_t at the given ctfa_ctfs-relative offset, or NULL if
    none, setting 'err' if non-NULL.  */
 static ctf_dict_t *
@@ -658,6 +730,25 @@ ctf_arc_open_by_name_sections (const ctf_archive_t *arc,
   return ctf_dict_open_sections (arc, symsect, strsect, name, errp);
 }
 
+/* Import the parent into a ctf archive, if this is a child, the parent is not
+   already set, and a suitable archive member exists.  No error is raised if
+   this is not possible: this is just a best-effort helper operation to give
+   people useful dicts to start with.  */
+static void
+ctf_arc_import_parent (const ctf_archive_t *arc, ctf_dict_t *fp)
+{
+  if ((fp->ctf_flags & LCTF_CHILD) && fp->ctf_parname && !fp->ctf_parent)
+    {
+      ctf_dict_t *parent = ctf_dict_open_cached ((ctf_archive_t *) arc,
+						 fp->ctf_parname, NULL);
+      if (parent)
+	{
+	  ctf_import (fp, parent);
+	  ctf_dict_close (parent);
+	}
+    }
+}
+
 /* Return the number of members in an archive.  */
 size_t
 ctf_archive_count (const ctf_archive_t *wrapper)
@@ -668,6 +759,139 @@ ctf_archive_count (const ctf_archive_t *wrapper)
   return wrapper->ctfi_archive->ctfa_ndicts;
 }
 
+/* Look up a symbol in an archive.  Return the dict in the archive that the
+   symbol is found in, and (optionally) the ctf_id_t of the symbol in that dict
+   (so you don't have to look it up yourself).  The dict and mapping are both
+   cached, so repeated lookups are nearly free.
+
+   As usual, you should ctf_dict_close() the returned dict once you are done
+   with it.
+
+   Returns NULL on error, and an error in errp (if set).  */
+
+ctf_dict_t *
+ctf_arc_lookup_symbol (ctf_archive_t *wrapper, unsigned long symidx,
+		       ctf_id_t *typep, int *errp)
+{
+  ctf_dict_t *fp;
+  ctf_id_t type;
+
+  /* The usual non-archive-transparent-wrapper special case.  */
+  if (!wrapper->ctfi_is_archive)
+    {
+      if ((type = ctf_lookup_by_symbol (wrapper->ctfi_dict, symidx)) == CTF_ERR)
+	{
+	  if (errp)
+	    *errp = ctf_errno (wrapper->ctfi_dict);
+	  return NULL;
+	}
+      if (typep)
+	*typep = type;
+      wrapper->ctfi_dict->ctf_refcnt++;
+      return wrapper->ctfi_dict;
+    }
+
+  if (wrapper->ctfi_symsect.cts_name == NULL
+      || wrapper->ctfi_symsect.cts_data == NULL
+      || wrapper->ctfi_symsect.cts_size == 0
+      || wrapper->ctfi_symsect.cts_entsize == 0)
+    {
+      if (errp)
+	*errp = ECTF_NOSYMTAB;
+      return NULL;
+    }
+
+  /* Make enough space for all possible symbols, if not already done.
+     We cache both the ctf_id_t and the originating dictionary of all symbols.
+     The dict links are weak, to the dictionaries cached in ctfi_dicts: their
+     refcnts are *not* bumped.  */
+
+  if (!wrapper->ctfi_syms)
+    {
+      if ((wrapper->ctfi_syms = calloc (wrapper->ctfi_symsect.cts_size
+					/ wrapper->ctfi_symsect.cts_entsize,
+					sizeof (ctf_id_t))) == NULL)
+	{
+	  if (errp)
+	    *errp = ENOMEM;
+	  return NULL;
+	}
+    }
+  if (!wrapper->ctfi_symdicts)
+    {
+      if ((wrapper->ctfi_symdicts = calloc (wrapper->ctfi_symsect.cts_size
+					    / wrapper->ctfi_symsect.cts_entsize,
+					    sizeof (ctf_dict_t *))) == NULL)
+	{
+	  if (errp)
+	    *errp = ENOMEM;
+	  return NULL;
+	}
+    }
+
+  /* Perhaps it's cached.  */
+  if (wrapper->ctfi_symdicts[symidx] != NULL)
+    {
+      if (wrapper->ctfi_symdicts[symidx] == &enosym)
+	{
+	  if (errp)
+	    *errp = ECTF_NOTYPEDAT;
+	  if (typep)
+	    *typep = CTF_ERR;
+	  return NULL;
+	}
+
+      if (typep)
+	*typep = wrapper->ctfi_syms[symidx];
+      wrapper->ctfi_symdicts[symidx]->ctf_refcnt++;
+      return wrapper->ctfi_symdicts[symidx];
+    }
+
+  /* Not cached: find it and cache it.  We must track open errors ourselves even
+     if our caller doesn't, to be able to distinguish no-error end-of-iteration
+     from open errors.  */
+
+  int local_err;
+  int *local_errp;
+  ctf_next_t *i = NULL;
+  const char *name;
+
+  if (errp)
+    local_errp = errp;
+  else
+    local_errp = &local_err;
+
+  while ((fp = ctf_archive_next (wrapper, &i, &name, 0, local_errp)) != NULL)
+    {
+      if ((type = ctf_lookup_by_symbol (fp, symidx)) != CTF_ERR)
+	{
+	  wrapper->ctfi_syms[symidx] = type;
+	  wrapper->ctfi_symdicts[symidx] = fp;
+	  ctf_next_destroy (i);
+
+	  if (typep)
+	    *typep = type;
+	  return fp;
+	}
+      ctf_dict_close (fp);
+    }
+  if (*local_errp != ECTF_NEXT_END)
+    {
+      ctf_next_destroy (i);
+      return NULL;
+    }
+  /* Don't leak end-of-iteration to the caller.  */
+  *local_errp = 0;
+
+  wrapper->ctfi_symdicts[symidx] = &enosym;
+
+  if (errp)
+    *errp = ECTF_NOTYPEDAT;
+  if (typep)
+    *typep = CTF_ERR;
+  return NULL;
+}
+
 /* Raw iteration over all CTF files in an archive.  We pass the raw data for all
    CTF files in turn to the specified callback function.  */
 static int
@@ -741,6 +965,7 @@ ctf_archive_iter_internal (const ctf_archive_t *wrapper,
 	return rc;
 
       f->ctf_archive = (ctf_archive_t *) wrapper;
+      ctf_arc_import_parent (wrapper, f);
       if ((rc = func (f, name, data)) != 0)
 	{
 	  ctf_dict_close (f);
@@ -779,6 +1004,8 @@ ctf_archive_iter (const ctf_archive_t *arc, ctf_archive_member_f *func,
    whether they are skipped or not, the caller must ctf_import the parent if
    need be.
 
+   The archive member is cached for rapid return on future calls.
+
    We identify parents by name rather than by flag value: for now, with the
    linker only emitting parents named _CTF_SECTION, this works well enough.  */
 
@@ -841,9 +1068,6 @@ ctf_archive_next (const ctf_archive_t *wrapper, ctf_next_t **it, const char **na
      is the parent (i.e. at most two iterations, but possibly an early return if
      *all* we have is a parent).  */
 
-  const ctf_sect_t *symsect;
-  const ctf_sect_t *strsect;
-
   do
     {
       if ((!wrapper->ctfi_is_archive) || (i->ctn_n >= le64toh (arc->ctfa_ndicts)))
@@ -855,14 +1079,6 @@ ctf_archive_next (const ctf_archive_t *wrapper, ctf_next_t **it, const char **na
 	  return NULL;
 	}
 
-      symsect = &wrapper->ctfi_symsect;
-      strsect = &wrapper->ctfi_strsect;
-
-      if (symsect->cts_name == NULL)
-	symsect = NULL;
-      if (strsect->cts_name == NULL)
-	strsect = NULL;
-
       modent = (ctf_archive_modent_t *) ((char *) arc
 					 + sizeof (struct ctf_archive));
       nametbl = (((const char *) arc) + le64toh (arc->ctfa_names));
@@ -874,8 +1090,7 @@ ctf_archive_next (const ctf_archive_t *wrapper, ctf_next_t **it, const char **na
   if (name)
     *name = name_;
 
-  f = ctf_dict_open_internal (arc, symsect, strsect, name_, errp);
-  f->ctf_archive = (ctf_archive_t *) wrapper;
+  f = ctf_dict_open_cached ((ctf_archive_t *) wrapper, name_, errp);
   return f;
 }
 
diff --git a/libctf/ctf-impl.h b/libctf/ctf-impl.h
index 62ea360..a9f7245 100644
--- a/libctf/ctf-impl.h
+++ b/libctf/ctf-impl.h
@@ -502,12 +502,15 @@ struct ctf_archive_internal
   int ctfi_unmap_on_close;
   ctf_dict_t *ctfi_dict;
   struct ctf_archive *ctfi_archive;
+  ctf_dynhash_t *ctfi_dicts;	  /* Dicts we have opened and cached.  */
+  ctf_dict_t **ctfi_symdicts;	  /* Array of index -> ctf_dict_t *.  */
+  ctf_id_t *ctfi_syms;		  /* Array of index -> ctf_id_t.  */
   ctf_sect_t ctfi_symsect;
   ctf_sect_t ctfi_strsect;
   int ctfi_free_symsect;
   int ctfi_free_strsect;
   void *ctfi_data;
-  bfd *ctfi_abfd;		    /* Optional source of section data.  */
+  bfd *ctfi_abfd;		  /* Optional source of section data.  */
   void (*ctfi_bfd_close) (struct ctf_archive_internal *);
 };
 
diff --git a/libctf/libctf.ver b/libctf/libctf.ver
index 7369d63..f0633f2 100644
--- a/libctf/libctf.ver
+++ b/libctf/libctf.ver
@@ -187,4 +187,7 @@ LIBCTF_1.1 {
 	ctf_add_func_sym;
 
 	ctf_link_add_linker_symbol;
+
+	ctf_arc_lookup_symbol;
+	ctf_arc_flush_caches;
 } LIBCTF_1.0;