libctf: Sanitize error types for PR 30836

Made sure there is no implicit conversion between signed and unsigned
return value for functions setting the ctf_errno value.
An example of the problem is that in ctf_member_next, the "offset" value
is either 0L or (ctf_id_t)-1L, but it should have been 0L or -1L.
The issue was discovered while building a 64 bit ld binary to be
executed on the Windows platform.
Example object file that demonstrates the issue is attached in the PR.

libctf/
	Affected functions adjusted.

Signed-off-by: Torbjörn SVENSSON <torbjorn.svensson@foss.st.com>
Co-Authored-By: Yvan ROUX <yvan.roux@foss.st.com>

1 files changed, 18 insertions, 21 deletions

diff --git a/libctf/ctf-lookup.c b/libctf/ctf-lookup.c
index c658491..177cb1b 100644
--- a/libctf/ctf-lookup.c
+++ b/libctf/ctf-lookup.c
@@ -143,7 +143,7 @@ ctf_lookup_by_name_internal (ctf_dict_t *fp, ctf_dict_t *child,
   ctf_id_t ntype, ptype;
 
   if (name == NULL)
-    return (ctf_set_errno (fp, EINVAL));
+    return (ctf_set_typed_errno (fp, EINVAL));
 
   for (p = name, end = name + strlen (name); *p != '\0'; p = q)
     {
@@ -273,10 +273,7 @@ ctf_lookup_by_name_internal (ctf_dict_t *fp, ctf_dict_t *child,
 		  free (fp->ctf_tmp_typeslice);
 		  fp->ctf_tmp_typeslice = xstrndup (p, (size_t) (q - p));
 		  if (fp->ctf_tmp_typeslice == NULL)
-		    {
-		      ctf_set_errno (fp, ENOMEM);
-		      return CTF_ERR;
-		    }
+		    return ctf_set_typed_errno (fp, ENOMEM);
 		}
 
 	      if ((type = ctf_lookup_by_rawhash (fp, lp->ctl_hash,
@@ -292,7 +289,7 @@ ctf_lookup_by_name_internal (ctf_dict_t *fp, ctf_dict_t *child,
     }
 
   if (*p != '\0' || type == 0)
-    return (ctf_set_errno (fp, ECTF_SYNTAX));
+    return (ctf_set_typed_errno (fp, ECTF_SYNTAX));
 
   return type;
 
@@ -306,13 +303,13 @@ ctf_lookup_by_name_internal (ctf_dict_t *fp, ctf_dict_t *child,
       if (fp->ctf_pptrtab_typemax < fp->ctf_typemax)
 	{
 	  if (refresh_pptrtab (fp, fp->ctf_parent) < 0)
-	    return -1;			/* errno is set for us.  */
+	    return CTF_ERR;			/* errno is set for us.  */
 	}
 
       if ((ptype = ctf_lookup_by_name_internal (fp->ctf_parent, fp,
 						name)) != CTF_ERR)
 	return ptype;
-      return (ctf_set_errno (fp, ctf_errno (fp->ctf_parent)));
+      return (ctf_set_typed_errno (fp, ctf_errno (fp->ctf_parent)));
     }
 
   return CTF_ERR;
@@ -407,10 +404,10 @@ ctf_lookup_variable (ctf_dict_t *fp, const char *name)
 
           if ((ptype = ctf_lookup_variable (fp->ctf_parent, name)) != CTF_ERR)
             return ptype;
-          return (ctf_set_errno (fp, ctf_errno (fp->ctf_parent)));
+          return (ctf_set_typed_errno (fp, ctf_errno (fp->ctf_parent)));
         }
 
-      return (ctf_set_errno (fp, ECTF_NOTYPEDAT));
+      return (ctf_set_typed_errno (fp, ECTF_NOTYPEDAT));
     }
 
   return ent->ctv_type;
@@ -673,7 +670,7 @@ ctf_symbol_next (ctf_dict_t *fp, ctf_next_t **it, const char **name,
   if (!i)
     {
       if ((i = ctf_next_create ()) == NULL)
-	return ctf_set_errno (fp, ENOMEM);
+	return ctf_set_typed_errno (fp, ENOMEM);
 
       i->cu.ctn_fp = fp;
       i->ctn_iter_fun = (void (*) (void)) ctf_symbol_next;
@@ -682,10 +679,10 @@ ctf_symbol_next (ctf_dict_t *fp, ctf_next_t **it, const char **name,
     }
 
   if ((void (*) (void)) ctf_symbol_next != i->ctn_iter_fun)
-    return (ctf_set_errno (fp, ECTF_NEXT_WRONGFUN));
+    return (ctf_set_typed_errno (fp, ECTF_NEXT_WRONGFUN));
 
   if (fp != i->cu.ctn_fp)
-    return (ctf_set_errno (fp, ECTF_NEXT_WRONGFP));
+    return (ctf_set_typed_errno (fp, ECTF_NEXT_WRONGFP));
 
   /* We intentionally use raw access, not ctf_lookup_by_symbol, to avoid
      incurring additional sorting cost for unsorted symtypetabs coming from the
@@ -701,7 +698,7 @@ ctf_symbol_next (ctf_dict_t *fp, ctf_next_t **it, const char **name,
       if (!dynh)
 	{
 	  ctf_next_destroy (i);
-	  return (ctf_set_errno (fp, ECTF_NEXT_END));
+	  return (ctf_set_typed_errno (fp, ECTF_NEXT_END));
 	}
 
       err = ctf_dynhash_next (dynh, &i->ctn_next, &dyn_name, &dyn_value);
@@ -710,7 +707,7 @@ ctf_symbol_next (ctf_dict_t *fp, ctf_next_t **it, const char **name,
 	{
 	  ctf_next_destroy (i);
 	  *it = NULL;
-	  return ctf_set_errno (fp, err);
+	  return ctf_set_typed_errno (fp, err);
 	}
 
       *name = dyn_name;
@@ -786,7 +783,7 @@ ctf_symbol_next (ctf_dict_t *fp, ctf_next_t **it, const char **name,
  end:
   ctf_next_destroy (i);
   *it = NULL;
-  return (ctf_set_errno (fp, ECTF_NEXT_END));
+  return (ctf_set_typed_errno (fp, ECTF_NEXT_END));
 }
 
 /* A bsearch function for function and object index names.  */
@@ -821,7 +818,7 @@ ctf_try_lookup_indexed (ctf_dict_t *fp, unsigned long symidx,
 	       "indexed symtypetab\n", symidx, symname);
 
   if (symname[0] == '\0')
-    return -1;					/* errno is set for us.  */
+    return CTF_ERR;					/* errno is set for us.  */
 
   if (is_function)
     {
@@ -835,7 +832,7 @@ ctf_try_lookup_indexed (ctf_dict_t *fp, unsigned long symidx,
 	      == NULL)
 	    {
 	      ctf_err_warn (fp, 0, 0, _("cannot sort function symidx"));
-	      return -1;				/* errno is set for us.  */
+	      return CTF_ERR;				/* errno is set for us.  */
 	    }
 	}
       symtypetab = (uint32_t *) (fp->ctf_buf + hp->cth_funcoff);
@@ -855,7 +852,7 @@ ctf_try_lookup_indexed (ctf_dict_t *fp, unsigned long symidx,
 	      == NULL)
 	    {
 	      ctf_err_warn (fp, 0, 0, _("cannot sort object symidx"));
-	      return -1;				/* errno is set for us. */
+	      return CTF_ERR;				/* errno is set for us. */
 	    }
 	}
 
@@ -878,7 +875,7 @@ ctf_try_lookup_indexed (ctf_dict_t *fp, unsigned long symidx,
 
   /* Should be impossible, but be paranoid.  */
   if ((idx - sxlate) > (ptrdiff_t) nidx)
-    return (ctf_set_errno (fp, ECTF_CORRUPT));
+    return (ctf_set_typed_errno (fp, ECTF_CORRUPT));
 
   ctf_dprintf ("Symbol %lx (%s) is of type %x\n", symidx, symname,
 	       symtypetab[*idx]);
@@ -1014,7 +1011,7 @@ ctf_lookup_by_sym_or_name (ctf_dict_t *fp, unsigned long symidx,
       return ret;
     }
   else
-    return (ctf_set_errno (fp, err));
+    return (ctf_set_typed_errno (fp, err));
 }
 
 /* Given a symbol table index, return the type of the function or data object