expr: Fix up constant_byte_string bitfield handling [PR98366]

constant_byte_string now uses a convert_to_bytes function, which doesn't
handle bitfields at all (don't punt on them, just puts them into wrong bits
or bytes).  Furthermore, I don't see a reason why that function should exist
at all, it duplicates native_encode_initializer functionality.
Except that native_encode_initializer punted on flexible array members and 2
tests in the testsuite relied on constant_byte_string handling those.
So, this patch throws away convert_to_bytes, uses native_encode_initializer
instead, but teaches it to handle flexible array members (only in the
non-mask mode with off == -1 for now), furthermore, it adds various corner
case checks that the old implementation was missing (like that STRING_CSTs
use int as length and therefore we shouldn't try to build larger than that
strings, or that native_encode*/native_interpret* APIs require sane
host and target bytes (8-bit on both).

2020-12-19  Jakub Jelinek  <jakub@redhat.com>

	PR middle-end/98366
	* fold-const.c (native_encode_initializer): Don't try to
	memset more than total_bytes with off == -1 even if len is large.
	Handle flexible array member initializers if off == -1 and mask is
	NULL.
	* expr.c (convert_to_bytes): Remove.
	(constant_byte_string): Use native_encode_initializer instead of
	convert_to_bytes.  Remove extraneous semicolon.  Punt on various
	corner-cases the APIs don't handle, like sizes > INT_MAX,
	BITS_PER_UNIT != 8, CHAR_BIT != 8.

	* gcc.c-torture/execute/pr98366.c: New test.

1 files changed, 52 insertions, 11 deletions

diff --git a/gcc/fold-const.c b/gcc/fold-const.c
index b78f3ab..1694ba4 100644
--- a/gcc/fold-const.c
+++ b/gcc/fold-const.c
@@ -8197,7 +8197,7 @@ native_encode_initializer (tree init, unsigned char *ptr, int len,
 
 	  gcc_assert (TREE_CODE (type) == RECORD_TYPE || mask == NULL);
 	  if (ptr != NULL)
-	    memset (ptr, '\0', MIN (total_bytes - off, len));
+	    memset (ptr, '\0', MIN (total_bytes - o, len));
 	  for (cnt = 0; ; cnt++)
 	    {
 	      tree val = NULL_TREE, field = NULL_TREE;
@@ -8266,11 +8266,33 @@ native_encode_initializer (tree init, unsigned char *ptr, int len,
 	      if (TREE_CODE (TREE_TYPE (field)) == ARRAY_TYPE
 		  && TYPE_DOMAIN (TREE_TYPE (field))
 		  && ! TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (field))))
-		return 0;
-	      if (DECL_SIZE_UNIT (field) == NULL_TREE
-		  || !tree_fits_shwi_p (DECL_SIZE_UNIT (field)))
-		return 0;
-	      fieldsize = tree_to_shwi (DECL_SIZE_UNIT (field));
+		{
+		  if (mask || off != -1)
+		    return 0;
+		  if (val == NULL_TREE)
+		    continue;
+		  if (TREE_CODE (TREE_TYPE (val)) != ARRAY_TYPE)
+		    return 0;
+		  fieldsize = int_size_in_bytes (TREE_TYPE (val));
+		  if (fieldsize < 0
+		      || (int) fieldsize != fieldsize
+		      || (pos + fieldsize) > INT_MAX)
+		    return 0;
+		  if (pos + fieldsize > total_bytes)
+		    {
+		      if (ptr != NULL && total_bytes - o < len)
+			memset (ptr + (total_bytes - o),
+				'\0', MIN (pos + fieldsize - o, len));
+		      total_bytes = pos + fieldsize;
+		    }
+		}
+	      else
+		{
+		  if (DECL_SIZE_UNIT (field) == NULL_TREE
+		      || !tree_fits_shwi_p (DECL_SIZE_UNIT (field)))
+		    return 0;
+		  fieldsize = tree_to_shwi (DECL_SIZE_UNIT (field));
+		}
 	      if (fieldsize == 0)
 		continue;
 
@@ -8439,12 +8461,31 @@ native_encode_initializer (tree init, unsigned char *ptr, int len,
 		  || (pos >= off
 		      && (pos + fieldsize <= (HOST_WIDE_INT) off + len)))
 		{
-		  if (!native_encode_initializer (val, ptr ? ptr + pos - o
-							   : NULL,
-						  fieldsize,
-						  off == -1 ? -1 : 0,
-						  mask ? mask + pos : NULL))
+		  int fldsize = fieldsize;
+		  if (off == -1)
+		    {
+		      tree fld = DECL_CHAIN (field);
+		      while (fld)
+			{
+			  if (TREE_CODE (fld) == FIELD_DECL)
+			    break;
+			  fld = DECL_CHAIN (fld);
+			}
+		      if (fld == NULL_TREE)
+			fldsize = len - pos;
+		    }
+		  r = native_encode_initializer (val, ptr ? ptr + pos - o
+							  : NULL,
+						 fldsize,
+						 off == -1 ? -1 : 0,
+						 mask ? mask + pos : NULL);
+		  if (!r)
 		    return 0;
+		  if (off == -1
+		      && fldsize != fieldsize
+		      && r > fieldsize
+		      && pos + r > total_bytes)
+		    total_bytes = pos + r;
 		}
 	      else
 		{