libctf: Sanitize error types for PR 30836

Made sure there is no implicit conversion between signed and unsigned
return value for functions setting the ctf_errno value.
An example of the problem is that in ctf_member_next, the "offset" value
is either 0L or (ctf_id_t)-1L, but it should have been 0L or -1L.
The issue was discovered while building a 64 bit ld binary to be
executed on the Windows platform.
Example object file that demonstrates the issue is attached in the PR.

libctf/
	Affected functions adjusted.

Signed-off-by: Torbjörn SVENSSON <torbjorn.svensson@foss.st.com>
Co-Authored-By: Yvan ROUX <yvan.roux@foss.st.com>

1 files changed, 9 insertions, 16 deletions

diff --git a/libctf/ctf-dedup.c b/libctf/ctf-dedup.c
index 5fdddfd..43f44f1 100644
--- a/libctf/ctf-dedup.c
+++ b/libctf/ctf-dedup.c
@@ -1318,8 +1318,7 @@ ctf_dedup_mark_conflicting_hash (ctf_dict_t *fp, const char *hval)
   if (ctf_dynset_cinsert (d->cd_conflicting_types, hval) < 0)
     {
       ctf_dprintf ("Out of memory marking %s as conflicted\n", hval);
-      ctf_set_errno (fp, errno);
-      return -1;
+      return ctf_set_errno (fp, errno);
     }
 
   /* If any types cite this type, mark them conflicted too.  */
@@ -2451,18 +2450,12 @@ ctf_dedup_maybe_synthesize_forward (ctf_dict_t *output, ctf_dict_t *target,
     {
       if ((emitted_forward = ctf_add_forward (target, CTF_ADD_ROOT, name,
 					      fwdkind)) == CTF_ERR)
-	{
-	  ctf_set_errno (output, ctf_errno (target));
-	  return CTF_ERR;
-	}
+	return ctf_set_typed_errno (output, ctf_errno (target));
 
       if (ctf_dynhash_cinsert (td->cd_output_emission_conflicted_forwards,
 			       decorated, (void *) (uintptr_t)
 			       emitted_forward) < 0)
-	{
-	  ctf_set_errno (output, ENOMEM);
-	  return CTF_ERR;
-	}
+	return ctf_set_typed_errno (output, ENOMEM);
     }
   else
     emitted_forward = (ctf_id_t) (uintptr_t) v;
@@ -2525,7 +2518,7 @@ ctf_dedup_id_to_target (ctf_dict_t *output, ctf_dict_t *target,
   if ((input->ctf_flags & LCTF_CHILD) && (LCTF_TYPE_ISPARENT (input, id)))
     {
       if (!ctf_assert (output, parents[input_num] <= ninputs))
-	return -1;
+	return CTF_ERR;
       input = inputs[parents[input_num]];
       input_num = parents[input_num];
     }
@@ -2534,7 +2527,7 @@ ctf_dedup_id_to_target (ctf_dict_t *output, ctf_dict_t *target,
 			     CTF_DEDUP_GID (output, input_num, id));
 
   if (!ctf_assert (output, hval && td->cd_output_emission_hashes))
-    return -1;
+    return CTF_ERR;
 
   /* If this type is a conflicted tagged structure, union, or forward,
      substitute a synthetic forward instead, emitting it if need be.  Only do
@@ -2553,7 +2546,7 @@ ctf_dedup_id_to_target (ctf_dict_t *output, ctf_dict_t *target,
       ctf_set_errno (err_fp, ctf_errno (output));
       ctf_err_warn (err_fp, 0, 0, _("cannot add synthetic forward for type "
 				    "%i/%lx"), input_num, id);
-      return -1;
+      return CTF_ERR;
     default:
       return emitted_forward;
     }
@@ -2568,7 +2561,7 @@ ctf_dedup_id_to_target (ctf_dict_t *output, ctf_dict_t *target,
       ctf_dprintf ("Checking shared parent for target\n");
       if (!ctf_assert (output, (target != output)
 		       && (target->ctf_flags & LCTF_CHILD)))
-	return -1;
+	return CTF_ERR;
 
       target_id = ctf_dynhash_lookup (od->cd_output_emission_hashes, hval);
 
@@ -2582,13 +2575,13 @@ ctf_dedup_id_to_target (ctf_dict_t *output, ctf_dict_t *target,
 	  ctf_err_warn (err_fp, 0, ctf_errno (output),
 			_("cannot add synthetic forward for type %i/%lx"),
 			input_num, id);
-	  return ctf_set_errno (err_fp, ctf_errno (output));
+	  return ctf_set_typed_errno (err_fp, ctf_errno (output));
 	default:
 	  return emitted_forward;
 	}
     }
   if (!ctf_assert (output, target_id))
-    return -1;
+    return CTF_ERR;
   return (ctf_id_t) (uintptr_t) target_id;
 }