libctf: fix ctf_open endianness problems with raw CTF files

ctf_open (or, rather, ctf_fdopen, which underlies it) has several
endianness problems, even though it was written after the
endian-swapping code was implemented, so should have been endian-aware.

Even though the comment right above the relevant check says that it wil
check for CTF magic in any endianness, it only checks in the native
endianness, so opening raw LE CTF files on BE, or vice-versa, will fail.
It also checks the CTF version by hand, without ever endianness-swapping
the header, so that too will fail, and is entirely redundant because
ctf_simple_open does the job properly in any case.  We have a similar
problem in the next if block, which checks for raw CTF archives: we are
checking in the native endianness while we should be doing a le64toh()
on it to check in little-endian form only: so opening CTF archives
created on the local machine will fail if the local machine is
big-endian.

Adding insult to injury, if ctf_simple_open then fails, we go on and try
to turn it into a single-element CTF archive regardless, throwing the
error away.  Since this involves dereferencing null pointers it is not
likely to work very well.

libctf/
	* ctf-open-bfd.c: Add swap.h and ctf-endian.h.
	(ctf_fdopen): Check for endian-swapped raw CTF magic, and
	little-endian CTF archive magic.  Do not check the CTF version:
	ctf_simple_open does that in endian-safe ways.  Do not dereference
	null pointers on open failure.

1 files changed, 14 insertions, 9 deletions

diff --git a/libctf/ctf-open-bfd.c b/libctf/ctf-open-bfd.c
index 76b7f9d..bb2d7e6 100644
--- a/libctf/ctf-open-bfd.c
+++ b/libctf/ctf-open-bfd.c
@@ -26,6 +26,8 @@
 #include <fcntl.h>
 #include <elf.h>
 #include <bfd.h>
+#include "swap.h"
+#include "ctf-endian.h"
 
 #include "elf-bfd.h"
 
@@ -243,24 +245,27 @@ ctf_fdopen (int fd, const char *filename, const char *target, int *errp)
   if ((nbytes = ctf_pread (fd, &ctfhdr, sizeof (ctfhdr), 0)) <= 0)
     return (ctf_set_open_errno (errp, nbytes < 0 ? errno : ECTF_FMT));
 
-  /* If we have read enough bytes to form a CTF header and the magic
-     string matches, attempt to interpret the file as raw CTF.  */
+  /* If we have read enough bytes to form a CTF header and the magic string
+     matches, in either endianness, attempt to interpret the file as raw
+     CTF.  */
 
-  if ((size_t) nbytes >= sizeof (ctf_preamble_t) &&
-      ctfhdr.ctp_magic == CTF_MAGIC)
+  if ((size_t) nbytes >= sizeof (ctf_preamble_t)
+      && (ctfhdr.ctp_magic == CTF_MAGIC
+	  || ctfhdr.ctp_magic == bswap_16 (CTF_MAGIC)))
     {
       ctf_file_t *fp = NULL;
       void *data;
 
-      if (ctfhdr.ctp_version > CTF_VERSION)
-	return (ctf_set_open_errno (errp, ECTF_CTFVERS));
-
       if ((data = ctf_mmap (st.st_size, 0, fd)) == NULL)
 	return (ctf_set_open_errno (errp, errno));
 
       if ((fp = ctf_simple_open (data, (size_t) st.st_size, NULL, 0, 0,
 				 NULL, 0, errp)) == NULL)
-	ctf_munmap (data, (size_t) st.st_size);
+	{
+	  ctf_munmap (data, (size_t) st.st_size);
+	  return NULL;			/* errno is set for us.  */
+	}
+
       fp->ctf_data_mmapped = data;
       fp->ctf_data_mmapped_len = (size_t) st.st_size;
 
@@ -270,7 +275,7 @@ ctf_fdopen (int fd, const char *filename, const char *target, int *errp)
   if ((nbytes = ctf_pread (fd, &arc_magic, sizeof (arc_magic), 0)) <= 0)
     return (ctf_set_open_errno (errp, nbytes < 0 ? errno : ECTF_FMT));
 
-  if ((size_t) nbytes >= sizeof (uint64_t) && arc_magic == CTFA_MAGIC)
+  if ((size_t) nbytes >= sizeof (uint64_t) && le64toh (arc_magic) == CTFA_MAGIC)
     {
       struct ctf_archive *arc;