wide-int: Fix estimation of buffer sizes for wide_int printing [PR111800]

As mentioned in the PR, my estimations on needed buffer size for wide_int
and especially widest_int printing were incorrect, I've used get_len ()
in the estimations, but that is true only for !wi::neg_p (x) values.
Under the hood, we have 3 ways to print numbers.
print_decs which if
  if ((wi.get_precision () <= HOST_BITS_PER_WIDE_INT)
      || (wi.get_len () == 1))
uses sprintf which always fits into WIDE_INT_PRINT_BUFFER_SIZE (positive or
negative) and otherwise uses print_hex,
print_decu which if
  if ((wi.get_precision () <= HOST_BITS_PER_WIDE_INT)
      || (wi.get_len () == 1 && !wi::neg_p (wi)))
uses sprintf which always fits into WIDE_INT_PRINT_BUFFER_SIZE (positive
only) and print_hex, which doesn't print most significant limbs which are
zero and the first limb which is non-zero prints such that redundant 0
hex digits aren't printed, while all limbs below that are printed with
"%016" PRIx64.  For wi::neg_p (x) values, the first limb of the precision
is always non-zero, so we print all the limbs for the precision.
So, the current estimations are accurate if !wi::neg_p (x), or when
print_decs will be used and x.get_len () == 1, otherwise we need to use
estimation based on get_precision () rather than get_len ().

I've introduced new inlines print_{dec{,s,u},hex}_buf_size which compute the
needed buffer length in bytes and return true if WIDE_INT_PRINT_BUFFER_SIZE
isn't sufficient and caller should XALLOCAVEC the buffer.

2023-10-15  Jakub Jelinek  <jakub@redhat.com>

	PR tree-optimization/111800
gcc/
	* wide-int-print.h (print_dec_buf_size, print_decs_buf_size,
	print_decu_buf_size, print_hex_buf_size): New inline functions.
	* wide-int.cc (assert_deceq): Use print_dec_buf_size.
	(assert_hexeq): Use print_hex_buf_size.
	* wide-int-print.cc (print_decs): Use print_decs_buf_size.
	(print_decu): Use print_decu_buf_size.
	(print_hex): Use print_hex_buf_size.
	(pp_wide_int_large): Use print_dec_buf_size.
	* value-range.cc (irange_bitmask::dump): Use print_hex_buf_size.
	* value-range-pretty-print.cc (vrange_printer::print_irange_bitmasks):
	Likewise.
	* tree-ssa-loop-niter.cc (do_warn_aggressive_loop_optimizations): Use
	print_dec_buf_size.  Use TYPE_SIGN macro in print_dec call argument.
gcc/c-family/
	* c-warn.cc (match_case_to_enum_1): Assert w.get_precision ()
	is smaller or equal to WIDE_INT_MAX_INL_PRECISION rather than
	w.get_len () is smaller or equal to WIDE_INT_MAX_INL_ELTS.

7 files changed, 69 insertions, 33 deletions

diff --git a/gcc/c-family/c-warn.cc b/gcc/c-family/c-warn.cc
index 3e38a86..3e2d02a 100644
--- a/gcc/c-family/c-warn.cc
+++ b/gcc/c-family/c-warn.cc
@@ -1519,7 +1519,7 @@ match_case_to_enum_1 (tree key, tree type, tree label)
   char buf[WIDE_INT_PRINT_BUFFER_SIZE];
   wide_int w = wi::to_wide (key);
 
-  gcc_assert (w.get_len () <= WIDE_INT_MAX_INL_ELTS);
+  gcc_assert (w.get_precision () <= WIDE_INT_MAX_INL_PRECISION);
   if (tree_fits_uhwi_p (key))
     print_dec (w, buf, UNSIGNED);
   else if (tree_fits_shwi_p (key))
diff --git a/gcc/tree-ssa-loop-niter.cc b/gcc/tree-ssa-loop-niter.cc
index a0a6b27..718582a 100644
--- a/gcc/tree-ssa-loop-niter.cc
+++ b/gcc/tree-ssa-loop-niter.cc
@@ -3874,13 +3874,13 @@ do_warn_aggressive_loop_optimizations (class loop *loop,
 
   gimple *estmt = last_nondebug_stmt (e->src);
   char buf[WIDE_INT_PRINT_BUFFER_SIZE], *p;
-  unsigned len = i_bound.get_len ();
-  if (len > WIDE_INT_MAX_INL_ELTS)
-    p = XALLOCAVEC (char, len * HOST_BITS_PER_WIDE_INT / 4 + 4);
+  unsigned len;
+  if (print_dec_buf_size (i_bound, TYPE_SIGN (TREE_TYPE (loop->nb_iterations)),
+			  &len))
+    p = XALLOCAVEC (char, len);
   else
     p = buf;
-  print_dec (i_bound, p, TYPE_UNSIGNED (TREE_TYPE (loop->nb_iterations))
-	     ? UNSIGNED : SIGNED);
+  print_dec (i_bound, p, TYPE_SIGN (TREE_TYPE (loop->nb_iterations)));
   auto_diagnostic_group d;
   if (warning_at (gimple_location (stmt), OPT_Waggressive_loop_optimizations,
 		  "iteration %s invokes undefined behavior", p))
diff --git a/gcc/value-range-pretty-print.cc b/gcc/value-range-pretty-print.cc
index 26173b8..4e43107 100644
--- a/gcc/value-range-pretty-print.cc
+++ b/gcc/value-range-pretty-print.cc
@@ -100,11 +100,10 @@ vrange_printer::print_irange_bitmasks (const irange &r) const
 
   pp_string (pp, " MASK ");
   char buf[WIDE_INT_PRINT_BUFFER_SIZE], *p;
-  unsigned len_mask = bm.mask ().get_len ();
-  unsigned len_val = bm.value ().get_len ();
-  unsigned len = MAX (len_mask, len_val);
-  if (len > WIDE_INT_MAX_INL_ELTS)
-    p = XALLOCAVEC (char, len * HOST_BITS_PER_WIDE_INT / 4 + 4);
+  unsigned len_mask, len_val;
+  if (print_hex_buf_size (bm.mask (), &len_mask)
+      | print_hex_buf_size (bm.value (), &len_val))
+    p = XALLOCAVEC (char, MAX (len_mask, len_val));
   else
     p = buf;
   print_hex (bm.mask (), p);
diff --git a/gcc/value-range.cc b/gcc/value-range.cc
index 333245e..f507ec5 100644
--- a/gcc/value-range.cc
+++ b/gcc/value-range.cc
@@ -251,11 +251,10 @@ irange_bitmask::dump (FILE *file) const
   pp_needs_newline (&buffer) = true;
   buffer.buffer->stream = file;
   pp_string (&buffer, "MASK ");
-  unsigned len_mask = m_mask.get_len ();
-  unsigned len_val = m_value.get_len ();
-  unsigned len = MAX (len_mask, len_val);
-  if (len > WIDE_INT_MAX_INL_ELTS)
-    p = XALLOCAVEC (char, len * HOST_BITS_PER_WIDE_INT / 4 + 4);
+  unsigned len_mask, len_val;
+  if (print_hex_buf_size (m_mask, &len_mask)
+      | print_hex_buf_size (m_value, &len_val))
+    p = XALLOCAVEC (char, MAX (len_mask, len_val));
   else
     p = buf;
   print_hex (m_mask, p);
diff --git a/gcc/wide-int-print.cc b/gcc/wide-int-print.cc
index ea6840b..5179525 100644
--- a/gcc/wide-int-print.cc
+++ b/gcc/wide-int-print.cc
@@ -75,9 +75,9 @@ void
 print_decs (const wide_int_ref &wi, FILE *file)
 {
   char buf[WIDE_INT_PRINT_BUFFER_SIZE], *p = buf;
-  unsigned len = wi.get_len ();
-  if (UNLIKELY (len > WIDE_INT_MAX_INL_ELTS))
-    p = XALLOCAVEC (char, len * HOST_BITS_PER_WIDE_INT / 4 + 4);
+  unsigned len;
+  if (print_decs_buf_size (wi, &len))
+    p = XALLOCAVEC (char, len);
   print_decs (wi, p);
   fputs (p, file);
 }
@@ -102,9 +102,9 @@ void
 print_decu (const wide_int_ref &wi, FILE *file)
 {
   char buf[WIDE_INT_PRINT_BUFFER_SIZE], *p = buf;
-  unsigned len = wi.get_len ();
-  if (UNLIKELY (len > WIDE_INT_MAX_INL_ELTS))
-    p = XALLOCAVEC (char, len * HOST_BITS_PER_WIDE_INT / 4 + 4);
+  unsigned len;
+  if (print_decu_buf_size (wi, &len))
+    p = XALLOCAVEC (char, len);
   print_decu (wi, p);
   fputs (p, file);
 }
@@ -141,9 +141,9 @@ void
 print_hex (const wide_int_ref &wi, FILE *file)
 {
   char buf[WIDE_INT_PRINT_BUFFER_SIZE], *p = buf;
-  unsigned len = wi.get_len ();
-  if (UNLIKELY (len > WIDE_INT_MAX_INL_ELTS))
-    p = XALLOCAVEC (char, len * HOST_BITS_PER_WIDE_INT / 4 + 4);
+  unsigned len;
+  if (print_hex_buf_size (wi, &len))
+    p = XALLOCAVEC (char, len);
   print_hex (wi, p);
   fputs (p, file);
 }
@@ -154,8 +154,10 @@ print_hex (const wide_int_ref &wi, FILE *file)
 void
 pp_wide_int_large (pretty_printer *pp, const wide_int_ref &w, signop sgn)
 {
-  unsigned int prec = w.get_precision ();
-  char *buf = XALLOCAVEC (char, (prec + 3) / 4 + 3);
+  unsigned int len;
+  if (!print_dec_buf_size (w, sgn, &len))
+    len = WIDE_INT_PRINT_BUFFER_SIZE;
+  char *buf = XALLOCAVEC (char, len);
   print_dec (w, buf, sgn);
   pp_string (pp, buf);
 }
diff --git a/gcc/wide-int-print.h b/gcc/wide-int-print.h
index a69cd5f..8dbfe24 100644
--- a/gcc/wide-int-print.h
+++ b/gcc/wide-int-print.h
@@ -36,4 +36,40 @@ extern void print_hex (const wide_int_ref &wi, char *buf);
 extern void print_hex (const wide_int_ref &wi, FILE *file);
 extern void pp_wide_int_large (pretty_printer *, const wide_int_ref &, signop);
 
+inline bool
+print_dec_buf_size (const wide_int_ref &wi, signop sgn, unsigned int *len)
+{
+  unsigned int l = wi.get_len ();
+  if ((l != 1 || sgn == UNSIGNED) && wi::neg_p (wi))
+    l = WIDE_INT_MAX_HWIS (wi.get_precision ());
+  l = l * HOST_BITS_PER_WIDE_INT / 4 + 4;
+  *len = l;
+  return UNLIKELY (l > WIDE_INT_PRINT_BUFFER_SIZE);
+}
+
+inline bool
+print_decs_buf_size (const wide_int_ref &wi, unsigned int *len)
+{
+  return print_dec_buf_size (wi, SIGNED, len);
+}
+
+inline bool
+print_decu_buf_size (const wide_int_ref &wi, unsigned int *len)
+{
+  return print_dec_buf_size (wi, UNSIGNED, len);
+}
+
+inline bool
+print_hex_buf_size (const wide_int_ref &wi, unsigned int *len)
+{
+  unsigned int l;
+  if (wi::neg_p (wi))
+    l = WIDE_INT_MAX_HWIS (wi.get_precision ());
+  else
+    l = wi.get_len ();
+  l = l * HOST_BITS_PER_WIDE_INT / 4 + 4;
+  *len = l;
+  return UNLIKELY (l > WIDE_INT_PRINT_BUFFER_SIZE);
+}
+
 #endif /* WIDE_INT_PRINT_H */
diff --git a/gcc/wide-int.cc b/gcc/wide-int.cc
index 6212a90..5426766 100644
--- a/gcc/wide-int.cc
+++ b/gcc/wide-int.cc
@@ -2450,9 +2450,9 @@ static void
 assert_deceq (const char *expected, const wide_int_ref &wi, signop sgn)
 {
   char buf[WIDE_INT_PRINT_BUFFER_SIZE], *p = buf;
-  unsigned len = wi.get_len ();
-  if (UNLIKELY (len > WIDE_INT_MAX_INL_ELTS))
-    p = XALLOCAVEC (char, len * HOST_BITS_PER_WIDE_INT / 4 + 4);
+  unsigned len;
+  if (print_dec_buf_size (wi, sgn, &len))
+    p = XALLOCAVEC (char, len);
   print_dec (wi, p, sgn);
   ASSERT_STREQ (expected, p);
 }
@@ -2463,9 +2463,9 @@ static void
 assert_hexeq (const char *expected, const wide_int_ref &wi)
 {
   char buf[WIDE_INT_PRINT_BUFFER_SIZE], *p = buf;
-  unsigned len = wi.get_len ();
-  if (UNLIKELY (len > WIDE_INT_MAX_INL_ELTS))
-    p = XALLOCAVEC (char, len * HOST_BITS_PER_WIDE_INT / 4 + 4);
+  unsigned len;
+  if (print_hex_buf_size (wi, &len))
+    p = XALLOCAVEC (char, len);
   print_hex (wi, p);
   ASSERT_STREQ (expected, p);
 }