i386: Fix ICEs on TImode signed overflow add/sub patterns [PR93376]

The following testcase ICEs, because during try_combine of i3:
(insn 18 17 19 2 (parallel [
            (set (reg:CCO 17 flags)
                (eq:CCO (plus:OI (sign_extend:OI (reg:TI 96))
                        (const_int 1 [0x1]))
                    (sign_extend:OI (plus:TI (reg:TI 96)
                            (const_int 1 [0x1])))))
            (set (reg:TI 98)
                (plus:TI (reg:TI 96)
                    (const_int 1 [0x1])))
        ]) "pr93376.c":8:10 223 {*addvti4_doubleword_1}
     (expr_list:REG_UNUSED (reg:TI 98)
        (expr_list:REG_DEAD (reg:TI 96)
            (nil))))
and i2:
(insn 17 37 18 2 (set (reg:TI 96)
        (const_wide_int 0x7fffffffffffffffffffffffffffffff)) "pr93376.c":8:10 65 {*movti_internal}
     (nil))
the eq in there gets simplified into:
(eq:CCO (const_wide_int 0x080000000000000000000000000000000)
    (const_wide_int 0x80000000000000000000000000000000))
and simplify-rtx.c tries to simplify it by simplifying MINUS
of the two operands.
Now, i386 defines MAX_BITSIZE_MODE_ANY_INT to 128, because OImode
and XImode are used mainly as a placeholder for the vector modes;
these new signed overflow patterns are an exception to that,
but what they really need is just TImode precision + 1 (maybe 2 worst case)
bits at any time.

wide-int.h defines WIDE_INT_MAX_ELTS in a way that it contains one more
HWI above number of HWIs to cover WIDE_INT_MAX_ELTS, so on i386 that is
3 HWIs, meaning that TImode precision + 1/2 bits is still representable in
there.  Unfortunately, the way wi::sub_large is implemented, it needs
not just those 3 HWIs, but one HWI above the maximum of the lengths of
both operands, which means it buffer overflows, overwrites the following
precision in wide_int_storage and ICEs later on.  The need for 4 HWIs is
only temporary, because canonize immediately after it canonicalizes it
back to 3 HWIs only.

The patch is something suggested by Richard S., avoid using OImode
for this and instead use a partial int mode that is smaller.

2020-01-23  Jakub Jelinek  <jakub@redhat.com>

	PR target/93376
	* config/i386/i386-modes.def (POImode): New mode.
	(MAX_BITSIZE_MODE_ANY_INT): Change from 128 to 160.
	* config/i386/i386.md (DPWI): New mode attribute.
	(addv<mode>4, subv<mode>4): Use <DPWI> instead of <DWI>.
	(QWI): Rename to...
	(QPWI): ... this.  Use POI instead of OI for TImode.
	(*addv<dwi>4_doubleword, *addv<dwi>4_doubleword_1,
	*subv<dwi>4_doubleword, *subv<dwi>4_doubleword_1): Use <QPWI>
	instead of <QWI>.

	* gcc.dg/pr93376.c: New test.

5 files changed, 77 insertions, 27 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index ac8b18f..3d50dab 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,16 @@
+2020-01-23  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/93376
+	* config/i386/i386-modes.def (POImode): New mode.
+	(MAX_BITSIZE_MODE_ANY_INT): Change from 128 to 160.
+	* config/i386/i386.md (DPWI): New mode attribute.
+	(addv<mode>4, subv<mode>4): Use <DPWI> instead of <DWI>.
+	(QWI): Rename to...
+	(QPWI): ... this.  Use POI instead of OI for TImode.
+	(*addv<dwi>4_doubleword, *addv<dwi>4_doubleword_1,
+	*subv<dwi>4_doubleword, *subv<dwi>4_doubleword_1): Use <QPWI>
+	instead of <QWI>.
+
 2020-01-23  Richard Sandiford  <richard.sandiford@arm.com>
 
 	PR target/93341
diff --git a/gcc/config/i386/i386-modes.def b/gcc/config/i386/i386-modes.def
index bb57801..4278bb8 100644
--- a/gcc/config/i386/i386-modes.def
+++ b/gcc/config/i386/i386-modes.def
@@ -107,10 +107,19 @@ INT_MODE (XI, 64);
 PARTIAL_INT_MODE (HI, 16, P2QI);
 PARTIAL_INT_MODE (SI, 32, P2HI);
 
+/* Mode used for signed overflow checking of TImode.  As
+   MAX_BITSIZE_MODE_ANY_INT is only 160, wide-int.h reserves only that
+   rounded up to multiple of HOST_BITS_PER_WIDE_INT bits in wide_int etc.,
+   so OImode is too large.  For the overflow checking we actually need
+   just 1 or 2 bits beyond TImode precision.  Use 160 bits to have
+   a multiple of 32.  */
+PARTIAL_INT_MODE (OI, 160, POI);
+
 /* Keep the OI and XI modes from confusing the compiler into thinking
    that these modes could actually be used for computation.  They are
-   only holders for vectors during data movement.  */
-#define MAX_BITSIZE_MODE_ANY_INT (128)
+   only holders for vectors during data movement.  Include POImode precision
+   though.  */
+#define MAX_BITSIZE_MODE_ANY_INT (160)
 
 /* The symbol Pmode stands for one of the above machine modes (usually SImode).
    The tm.h file specifies which one.  It is not a distinct mode.  */
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index b5b53bb..98d452a 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -6054,15 +6054,18 @@
   [(set_attr "type" "alu")
    (set_attr "mode" "QI")])
 
+;; Like DWI, but use POImode instead of OImode.
+(define_mode_attr DPWI [(QI "HI") (HI "SI") (SI "DI") (DI "TI") (TI "POI")])
+
 ;; Add with jump on overflow.
 (define_expand "addv<mode>4"
   [(parallel [(set (reg:CCO FLAGS_REG)
 		   (eq:CCO
-		     (plus:<DWI>
-		       (sign_extend:<DWI>
+		     (plus:<DPWI>
+		       (sign_extend:<DPWI>
 			 (match_operand:SWIDWI 1 "nonimmediate_operand"))
 		       (match_dup 4))
-			 (sign_extend:<DWI>
+			 (sign_extend:<DPWI>
 			   (plus:SWIDWI (match_dup 1)
 			     (match_operand:SWIDWI 2
 			       "<general_hilo_operand>")))))
@@ -6078,7 +6081,7 @@
   if (CONST_SCALAR_INT_P (operands[2]))
     operands[4] = operands[2];
   else
-    operands[4] = gen_rtx_SIGN_EXTEND (<DWI>mode, operands[2]);
+    operands[4] = gen_rtx_SIGN_EXTEND (<DPWI>mode, operands[2]);
 })
 
 (define_insn "*addv<mode>4"
@@ -6123,17 +6126,17 @@
 	      (const_string "<MODE_SIZE>")))])
 
 ;; Quad word integer modes as mode attribute.
-(define_mode_attr QWI [(SI "TI") (DI "OI")])
+(define_mode_attr QPWI [(SI "TI") (DI "POI")])
 
 (define_insn_and_split "*addv<dwi>4_doubleword"
   [(set (reg:CCO FLAGS_REG)
 	(eq:CCO
-	  (plus:<QWI>
-	    (sign_extend:<QWI>
+	  (plus:<QPWI>
+	    (sign_extend:<QPWI>
 	      (match_operand:<DWI> 1 "nonimmediate_operand" "%0,0"))
-	    (sign_extend:<QWI>
+	    (sign_extend:<QPWI>
 	      (match_operand:<DWI> 2 "x86_64_hilo_general_operand" "r<di>,o")))
-	  (sign_extend:<QWI>
+	  (sign_extend:<QPWI>
 	    (plus:<DWI> (match_dup 1) (match_dup 2)))))
    (set (match_operand:<DWI> 0 "nonimmediate_operand" "=ro,r")
 	(plus:<DWI> (match_dup 1) (match_dup 2)))]
@@ -6172,11 +6175,11 @@
 (define_insn_and_split "*addv<dwi>4_doubleword_1"
   [(set (reg:CCO FLAGS_REG)
 	(eq:CCO
-	  (plus:<QWI>
-	    (sign_extend:<QWI>
+	  (plus:<QPWI>
+	    (sign_extend:<QPWI>
 	      (match_operand:<DWI> 1 "nonimmediate_operand" "%0"))
-	    (match_operand:<QWI> 3 "const_scalar_int_operand" ""))
-	  (sign_extend:<QWI>
+	    (match_operand:<QPWI> 3 "const_scalar_int_operand" ""))
+	  (sign_extend:<QPWI>
 	    (plus:<DWI>
 	      (match_dup 1)
 	      (match_operand:<DWI> 2 "x86_64_hilo_general_operand" "<di>")))))
@@ -6570,11 +6573,11 @@
 (define_expand "subv<mode>4"
   [(parallel [(set (reg:CCO FLAGS_REG)
 		   (eq:CCO
-		     (minus:<DWI>
-		       (sign_extend:<DWI>
+		     (minus:<DPWI>
+		       (sign_extend:<DPWI>
 			 (match_operand:SWIDWI 1 "nonimmediate_operand"))
 		       (match_dup 4))
-		     (sign_extend:<DWI>
+		     (sign_extend:<DPWI>
 		       (minus:SWIDWI (match_dup 1)
 				     (match_operand:SWIDWI 2
 						"<general_hilo_operand>")))))
@@ -6590,7 +6593,7 @@
   if (CONST_SCALAR_INT_P (operands[2]))
     operands[4] = operands[2];
   else
-    operands[4] = gen_rtx_SIGN_EXTEND (<DWI>mode, operands[2]);
+    operands[4] = gen_rtx_SIGN_EXTEND (<DPWI>mode, operands[2]);
 })
 
 (define_insn "*subv<mode>4"
@@ -6637,12 +6640,12 @@
 (define_insn_and_split "*subv<dwi>4_doubleword"
   [(set (reg:CCO FLAGS_REG)
 	(eq:CCO
-	  (minus:<QWI>
-	    (sign_extend:<QWI>
+	  (minus:<QPWI>
+	    (sign_extend:<QPWI>
 	      (match_operand:<DWI> 1 "nonimmediate_operand" "0,0"))
-	    (sign_extend:<QWI>
+	    (sign_extend:<QPWI>
 	      (match_operand:<DWI> 2 "x86_64_hilo_general_operand" "r<di>,o")))
-	  (sign_extend:<QWI>
+	  (sign_extend:<QPWI>
 	    (minus:<DWI> (match_dup 1) (match_dup 2)))))
    (set (match_operand:<DWI> 0 "nonimmediate_operand" "=ro,r")
 	(minus:<DWI> (match_dup 1) (match_dup 2)))]
@@ -6679,11 +6682,11 @@
 (define_insn_and_split "*subv<dwi>4_doubleword_1"
   [(set (reg:CCO FLAGS_REG)
 	(eq:CCO
-	  (minus:<QWI>
-	    (sign_extend:<QWI>
+	  (minus:<QPWI>
+	    (sign_extend:<QPWI>
 	      (match_operand:<DWI> 1 "nonimmediate_operand" "0"))
-	    (match_operand:<QWI> 3 "const_scalar_int_operand" ""))
-	  (sign_extend:<QWI>
+	    (match_operand:<QPWI> 3 "const_scalar_int_operand" ""))
+	  (sign_extend:<QPWI>
 	    (minus:<DWI>
 	      (match_dup 1)
 	      (match_operand:<DWI> 2 "x86_64_hilo_general_operand" "<di>")))))
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 7a03ebb..fef5951 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,8 @@
+2020-01-23  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/93376
+	* gcc.dg/pr93376.c: New test.
+
 2020-01-23  Richard Sandiford  <richard.sandiford@arm.com>
 
 	PR target/93341
diff --git a/gcc/testsuite/gcc.dg/pr93376.c b/gcc/testsuite/gcc.dg/pr93376.c
new file mode 100644
index 0000000..49341ff
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr93376.c
@@ -0,0 +1,20 @@
+/* PR target/93376 */
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-Og -finline-functions-called-once" } */
+
+unsigned a, b, c;
+
+int
+bar (int x)
+{
+  short s = __builtin_sub_overflow (~x, 0, &b);
+  a = __builtin_ffsll (~x);
+  return __builtin_add_overflow_p (-(unsigned __int128) a, s,
+				   (unsigned __int128) 0);
+}
+
+void
+foo (void)
+{
+  c = bar (0);
+}