middle-end/100547 - check rtvec_alloc size

This makes the rtvec_alloc argument size_t catching overflow and
truncated arguments (from "invalid" testcases), verifying the
argument against INT_MAX which is the limit set by the int
typed rtvec_def.num_elem member.

2021-05-12  Richard Biener  <rguenther@suse.de>

	PR middle-end/100547
	* rtl.h (rtvec_alloc): Make argument size_t.
	* rtl.c (rtvec_alloc): Verify the count is less than INT_MAX.

2 files changed, 5 insertions, 2 deletions

diff --git a/gcc/rtl.c b/gcc/rtl.c
index 035eadc..b0ba1ff 100644
--- a/gcc/rtl.c
+++ b/gcc/rtl.c
@@ -158,10 +158,13 @@ static size_t rtvec_alloc_sizes;
    Store the length, and initialize all elements to zero.  */
 
 rtvec
-rtvec_alloc (int n)
+rtvec_alloc (size_t n)
 {
   rtvec rt;
 
+  /* rtvec_def.num_elem is an int.  */
+  gcc_assert (n < INT_MAX);
+
   rt = ggc_alloc_rtvec_sized (n);
   /* Clear out the vector.  */
   memset (&rt->elem[0], 0, n * sizeof (rtx));
diff --git a/gcc/rtl.h b/gcc/rtl.h
index c5f3d20..35178b5 100644
--- a/gcc/rtl.h
+++ b/gcc/rtl.h
@@ -2976,7 +2976,7 @@ extern rtx rtx_alloc_stat_v (RTX_CODE MEM_STAT_DECL, int);
 	       (sizeof (struct hwivec_def)			\
 		+ ((NWORDS)-1) * sizeof (HOST_WIDE_INT)))	\
 
-extern rtvec rtvec_alloc (int);
+extern rtvec rtvec_alloc (size_t);
 extern rtvec shallow_copy_rtvec (rtvec);
 extern bool shared_const_p (const_rtx);
 extern rtx copy_rtx (rtx);