libctf: a couple of small error-handling fixes

Out-of-memory errors initializing the string atoms table were
disregarded (though they would have caused a segfault very shortly
afterwards).  Errors hashing types during deduplication were only
reported if they happened on the output dict, which is almost never the
case (most errors are going to be on the dict we're working over, which
is going to be one of the inputs).  (The error was detected in both
cases, but the errno was extracted from the wrong dict.)

libctf/ChangeLog
2021-03-18  Nick Alcock  <nick.alcock@oracle.com>

	* ctf-dedup.c (ctf_dedup_rhash_type): Report errors on the input
	dict properly.
	* ctf-open.c (ctf_bufopen_internal): Report errors initializing
	the atoms table.

3 files changed, 25 insertions, 10 deletions

diff --git a/libctf/ChangeLog b/libctf/ChangeLog
index 85eb09d..2d80e78 100644
--- a/libctf/ChangeLog
+++ b/libctf/ChangeLog
@@ -1,5 +1,12 @@
 2021-03-18  Nick Alcock  <nick.alcock@oracle.com>
 
+	* ctf-dedup.c (ctf_dedup_rhash_type): Report errors on the input
+	dict properly.
+	* ctf-open.c (ctf_bufopen_internal): Report errors initializing
+	the atoms table.
+
+2021-03-18  Nick Alcock  <nick.alcock@oracle.com>
+
 	* ctf-impl.h (ctf_next_t) <ctn_tp>: New.
 	<u.ctn_mp>: Remove.
 	<u.ctn_lmp>: Remove.
diff --git a/libctf/ctf-dedup.c b/libctf/ctf-dedup.c
index b8a7d49..572a68d 100644
--- a/libctf/ctf-dedup.c
+++ b/libctf/ctf-dedup.c
@@ -572,7 +572,7 @@ ctf_dedup_rhash_type (ctf_dict_t *fp, ctf_dict_t *input, ctf_dict_t **inputs,
   char hashbuf[CTF_SHA1_SIZE];
   const char *hval = NULL;
   const char *whaterr;
-  int err;
+  int err = 0;
 
   const char *citer = NULL;
   ctf_dynset_t *citers = NULL;
@@ -697,7 +697,7 @@ ctf_dedup_rhash_type (ctf_dict_t *fp, ctf_dict_t *input, ctf_dict_t **inputs,
 	if (ctf_type_encoding (input, type, &ep) < 0)
 	  {
 	    whaterr = N_("error getting encoding");
-	    goto err;
+	    goto input_err;
 	  }
 	ctf_dedup_sha1_add (&hash, &ep, sizeof (ctf_encoding_t), "encoding",
 			    depth);
@@ -770,7 +770,7 @@ ctf_dedup_rhash_type (ctf_dict_t *fp, ctf_dict_t *input, ctf_dict_t **inputs,
 	if (ctf_array_info (input, type, &ar) < 0)
 	  {
 	    whaterr = N_("error getting array info");
-	    goto err;
+	    goto input_err;
 	  }
 
 	if ((hval = ctf_dedup_hash_type (fp, input, inputs, parents, input_num,
@@ -808,7 +808,7 @@ ctf_dedup_rhash_type (ctf_dict_t *fp, ctf_dict_t *input, ctf_dict_t **inputs,
 	if (ctf_func_type_info (input, type, &fi) < 0)
 	  {
 	    whaterr = N_("error getting func type info");
-	    goto err;
+	    goto input_err;
 	  }
 
 	if ((hval = ctf_dedup_hash_type (fp, input, inputs, parents, input_num,
@@ -828,6 +828,7 @@ ctf_dedup_rhash_type (ctf_dict_t *fp, ctf_dict_t *input, ctf_dict_t **inputs,
 
 	if ((args = calloc (fi.ctc_argc, sizeof (ctf_id_t))) == NULL)
 	  {
+	    err = ENOMEM;
 	    whaterr = N_("error doing memory allocation");
 	    goto err;
 	  }
@@ -836,7 +837,7 @@ ctf_dedup_rhash_type (ctf_dict_t *fp, ctf_dict_t *input, ctf_dict_t **inputs,
 	  {
 	    free (args);
 	    whaterr = N_("error getting func arg type");
-	    goto err;
+	    goto input_err;
 	  }
 	for (j = 0; j < fi.ctc_argc; j++)
 	  {
@@ -871,7 +872,7 @@ ctf_dedup_rhash_type (ctf_dict_t *fp, ctf_dict_t *input, ctf_dict_t **inputs,
 	if (ctf_errno (input) != ECTF_NEXT_END)
 	  {
 	    whaterr = N_("error doing enum member iteration");
-	    goto err;
+	    goto input_err;
 	  }
 	break;
       }
@@ -916,7 +917,7 @@ ctf_dedup_rhash_type (ctf_dict_t *fp, ctf_dict_t *input, ctf_dict_t **inputs,
 	if (ctf_errno (input) != ECTF_NEXT_END)
 	  {
 	    whaterr = N_("error doing struct/union member iteration");
-	    goto err;
+	    goto input_err;
 	  }
 	break;
       }
@@ -971,10 +972,12 @@ ctf_dedup_rhash_type (ctf_dict_t *fp, ctf_dict_t *input, ctf_dict_t **inputs,
 
  iterr:
   ctf_next_destroy (i);
+ input_err:
+  err = ctf_errno (input);
  err:
   ctf_sha1_fini (&hash, NULL);
-  ctf_err_warn (fp, 0, 0, _("%s (%i): %s: during type hashing for type %lx, "
-			    "kind %i"), ctf_link_input_name (input),
+  ctf_err_warn (fp, 0, err, _("%s (%i): %s: during type hashing for type %lx, "
+			      "kind %i"), ctf_link_input_name (input),
 		input_num, gettext (whaterr), type, kind);
   return NULL;
  oom:
diff --git a/libctf/ctf-open.c b/libctf/ctf-open.c
index c2d9a33..8d11134 100644
--- a/libctf/ctf-open.c
+++ b/libctf/ctf-open.c
@@ -1545,7 +1545,12 @@ ctf_bufopen_internal (const ctf_sect_t *ctfsect, const ctf_sect_t *symsect,
      ctf_set_base().  */
 
   ctf_set_version (fp, hp, hp->cth_version);
-  ctf_str_create_atoms (fp);
+  if (ctf_str_create_atoms (fp) < 0)
+    {
+      err = ENOMEM;
+      goto bad;
+    }
+
   fp->ctf_parmax = CTF_MAX_PTYPE;
   memcpy (&fp->ctf_data, ctfsect, sizeof (ctf_sect_t));