libctf: mmappable archives

If you need to store a large number of CTF containers somewhere, this
provides a dedicated facility for doing so: an mmappable archive format
like a very simple tar or ar without all the system-dependent format
horrors or need for heavy file copying, with built-in compression of
files above a particular size threshold.

libctf automatically mmap()s uncompressed elements of these archives, or
uncompresses them, as needed.  (If the platform does not support mmap(),
copying into dynamically-allocated buffers is used.)

Archive iteration operations are partitioned into raw and non-raw
forms. Raw operations pass thhe raw archive contents to the callback:
non-raw forms open each member with ctf_bufopen() and pass the resulting
ctf_file_t to the iterator instead.  This lets you manipulate the raw
data in the archive, or the contents interpreted as a CTF file, as
needed.

It is not yet known whether we will store CTF archives in a linked ELF
object in one of these (akin to debugdata) or whether they'll get one
section per TU plus one parent container for types shared between them.
(In the case of ELF objects with very large numbers of TUs, an archive
of all of them would seem preferable, so we might just use an archive,
and add lzma support so you can assume that .gnu_debugdata and .ctf are
compressed using the same algorithm if both are present.)

To make usage easier, the ctf_archive_t is not the on-disk
representation but an abstraction over both ctf_file_t's and archives of
many ctf_file_t's: users see both CTF archives and raw CTF files as
ctf_archive_t's upon opening, the only difference being that a raw CTF
file has only a single "archive member", named ".ctf" (the default if a
null pointer is passed in as the name).  The next commit will make use
of this facility, in addition to providing the public interface to
actually open archives.  (In the future, it should be possible to have
all CTF sections in an ELF file appear as an "archive" in the same
fashion.)

This machinery is also used to allow library-internal creators of
ctf_archive_t's (such as the next commit) to stash away an ELF string
and symbol table, so that all opens of members in a given archive will
use them.  This lets CTF archives exploit the ELF string and symbol
table just like raw CTF files can.

(All this leads to somewhat confusing type naming.  The ctf_archive_t is
a typedef for the opaque internal type, struct ctf_archive_internal: the
non-internal "struct ctf_archive" is the on-disk structure meant for
other libraries manipulating CTF files.  It is probably clearest to use
the struct name for struct ctf_archive_internal inside the program, and
the typedef names outside.)

libctf/
	* ctf-archive.c: New.
	* ctf-impl.h (ctf_archive_internal): New type.
	(ctf_arc_open_internal): New declaration.
	(ctf_arc_bufopen): Likewise.
	(ctf_arc_close_internal): Likewise.
include/
	* ctf.h (CTFA_MAGIC): New.
	(struct ctf_archive): New.
	(struct ctf_archive_modent): Likewise.
	* ctf-api.h (ctf_archive_member_f): New.
	(ctf_archive_raw_member_f): Likewise.
	(ctf_arc_write): Likewise.
	(ctf_arc_close): Likewise.
	(ctf_arc_open_by_name): Likewise.
	(ctf_archive_iter): Likewise.
	(ctf_archive_raw_iter): Likewise.
	(ctf_get_arc): Likewise.

3 files changed, 91 insertions, 0 deletions

diff --git a/include/ChangeLog b/include/ChangeLog
index 8b82efe..fb7bd88 100644
--- a/include/ChangeLog
+++ b/include/ChangeLog
@@ -1,5 +1,19 @@
 2019-05-28  Nick Alcock  <nick.alcock@oracle.com>
 
+	* ctf.h (CTFA_MAGIC): New.
+	(struct ctf_archive): New.
+	(struct ctf_archive_modent): Likewise.
+	* ctf-api.h (ctf_archive_member_f): New.
+	(ctf_archive_raw_member_f): Likewise.
+	(ctf_arc_write): Likewise.
+	(ctf_arc_close): Likewise.
+	(ctf_arc_open_by_name): Likewise.
+	(ctf_archive_iter): Likewise.
+	(ctf_archive_raw_iter): Likewise.
+	(ctf_get_arc): Likewise.
+
+2019-05-28  Nick Alcock  <nick.alcock@oracle.com>
+
 	* ctf-api.h (ctf_file_close): New declaration.
 	(ctf_getdatasect): Likewise.
 	(ctf_parent_file): Likewise.
diff --git a/include/ctf-api.h b/include/ctf-api.h
index d9eff0b..4cac635 100644
--- a/include/ctf-api.h
+++ b/include/ctf-api.h
@@ -198,13 +198,35 @@ enum
 #define	CTF_ADD_NONROOT	0	/* Type only visible in nested scope.  */
 #define	CTF_ADD_ROOT	1	/* Type visible at top-level scope.  */
 
+/* These typedefs are used to define the signature for callback functions
+   that can be used with the iteration and visit functions below.  */
+
+typedef int ctf_archive_member_f (ctf_file_t *fp, const char *name, void *arg);
+typedef int ctf_archive_raw_member_f (const char *name, const void *content,
+				      size_t len, void *arg);
+
 extern ctf_sect_t ctf_getdatasect (const ctf_file_t *);
+extern ctf_archive_t *ctf_get_arc (const ctf_file_t *);
+extern void ctf_arc_close (ctf_archive_t *);
+extern ctf_file_t *ctf_arc_open_by_name (const ctf_archive_t *,
+					 const char *, int *);
+extern ctf_file_t *ctf_arc_open_by_name_sections (const ctf_archive_t *,
+						  const ctf_sect_t *,
+						  const ctf_sect_t *,
+						  const char *, int *);
+
+/* The next functions return or close real CTF files, or write out CTF archives,
+   not opaque containers around either.  */
+
 extern ctf_file_t *ctf_simple_open (const char *, size_t, const char *, size_t,
 				   size_t, const char *, size_t, int *);
 extern ctf_file_t *ctf_bufopen (const ctf_sect_t *, const ctf_sect_t *,
 				const ctf_sect_t *, int *);
 extern void ctf_file_close (ctf_file_t *);
 
+extern int ctf_arc_write (const char *, ctf_file_t **, size_t,
+			  const char **, size_t);
+
 extern ctf_file_t *ctf_parent_file (ctf_file_t *);
 extern const char *ctf_parent_name (ctf_file_t *);
 extern void ctf_parent_name_set (ctf_file_t *, const char *);
@@ -218,6 +240,14 @@ extern void *ctf_getspecific (ctf_file_t *);
 
 extern int ctf_errno (ctf_file_t *);
 extern const char *ctf_errmsg (int);
+extern int ctf_archive_iter (const ctf_archive_t *, ctf_archive_member_f *,
+			     void *);
+/* This function alone does not currently operate on CTF files masquerading
+   as archives, and returns -EINVAL: the raw data is no longer available.  It is
+   expected to be used only by archiving tools, in any case, which have no need
+   to deal with non-archives at all.  */
+extern int ctf_archive_raw_iter (const ctf_archive_t *,
+				 ctf_archive_raw_member_f *, void *);
 extern ctf_id_t ctf_add_array (ctf_file_t *, uint32_t,
 			       const ctf_arinfo_t *);
 extern ctf_id_t ctf_add_const (ctf_file_t *, uint32_t, ctf_id_t);
diff --git a/include/ctf.h b/include/ctf.h
index 6580a21..2c3384f 100644
--- a/include/ctf.h
+++ b/include/ctf.h
@@ -510,6 +510,53 @@ typedef struct ctf_enum
   int cte_value;		/* Value associated with this name.  */
 } ctf_enum_t;
 
+/* The ctf_archive is a collection of ctf_file_t's stored together. The format
+   is suitable for mmap()ing: this control structure merely describes the
+   mmap()ed archive (and overlaps the first few bytes of it), hence the
+   greater care taken with integral types.  All CTF files in an archive
+   must have the same data model.  (This is not validated.)
+
+   All integers in this structure are stored in little-endian byte order.
+
+   The code relies on the fact that everything in this header is a uint64_t
+   and thus the header needs no padding (in particular, that no padding is
+   needed between ctfa_ctfs and the unnamed ctfa_archive_modent array
+   that follows it).
+
+   This is *not* the same as the data structure returned by the ctf_arc_*()
+   functions:  this is the low-level on-disk representation.  */
+
+#define CTFA_MAGIC 0x8b47f2a4d7623eeb	/* Random.  */
+struct ctf_archive
+{
+  /* Magic number.  (In loaded files, overwritten with the file size
+     so ctf_arc_close() knows how much to munmap()).  */
+  uint64_t ctfa_magic;
+
+  /* CTF data model.  */
+  uint64_t ctfa_model;
+
+  /* Number of CTF files in the archive.  */
+  uint64_t ctfa_nfiles;
+
+  /* Offset of the name table.  */
+  uint64_t ctfa_names;
+
+  /* Offset of the CTF table.  Each element starts with a size (a uint64_t
+     in network byte order) then a ctf_file_t of that size.  */
+  uint64_t ctfa_ctfs;
+};
+
+/* An array of ctfa_nnamed of this structure lies at
+   ctf_archive[ctf_archive->ctfa_modents] and gives the ctfa_ctfs or
+   ctfa_names-relative offsets of each name or ctf_file_t.  */
+
+typedef struct ctf_archive_modent
+{
+  uint64_t name_offset;
+  uint64_t ctf_offset;
+} ctf_archive_modent_t;
+
 #ifdef	__cplusplus
 }
 #endif