ifcvt: Avoid ICEs trying to force_operand random RTL [PR97487]

As the testcase shows, RTL ifcvt can throw random RTL (whatever it found in
some insns) at expand_binop or expand_unop and expects it to do something
(and then will check if it created valid insns and punts if not).
These functions in the end if the operands don't match try to
copy_to_mode_reg the operands, which does
if (!general_operand (x, VOIDmode))
  x = force_operand (x, temp);
but, force_operand is far from handling all possible RTLs, it will ICE for
all more unusual RTL codes.  Basically handles just simple arithmetic and
unary RTL operations if they have an optab and
expand_simple_binop/expand_simple_unop ICE on others.

The following patch fixes it by adding some operand verification (whether
there is a hope that copy_to_mode_reg will succeed on those).  It is added
both to noce_emit_move_insn (not needed for this exact testcase,
that function simply tries to recog the insn as is and if it fails,
handles some simple binop/unop cases; the patch performs the verification
of their operands) and noce_try_sign_mask.

2021-02-03  Jakub Jelinek  <jakub@redhat.com>

	PR middle-end/97487
	* ifcvt.c (noce_can_force_operand): New function.
	(noce_emit_move_insn): Use it.
	(noce_try_sign_mask): Likewise.  Formatting fix.

	* gcc.dg/pr97487-1.c: New test.
	* gcc.dg/pr97487-2.c: New test.

3 files changed, 92 insertions, 6 deletions

diff --git a/gcc/ifcvt.c b/gcc/ifcvt.c
index 1480a94a..b467eb5 100644
--- a/gcc/ifcvt.c
+++ b/gcc/ifcvt.c
@@ -885,6 +885,60 @@ noce_emit_store_flag (struct noce_if_info *if_info, rtx x, int reversep,
 			   || code == GEU || code == GTU), normalize);
 }
 
+/* Return true if X can be safely forced into a register by copy_to_mode_reg
+   / force_operand.  */
+
+static bool
+noce_can_force_operand (rtx x)
+{
+  if (general_operand (x, VOIDmode))
+    return true;
+  if (SUBREG_P (x))
+    {
+      if (!noce_can_force_operand (SUBREG_REG (x)))
+	return false;
+      return true;
+    }
+  if (ARITHMETIC_P (x))
+    {
+      if (!noce_can_force_operand (XEXP (x, 0))
+	  || !noce_can_force_operand (XEXP (x, 1)))
+	return false;
+      switch (GET_CODE (x))
+	{
+	case MULT:
+	case DIV:
+	case MOD:
+	case UDIV:
+	case UMOD:
+	  return true;
+	default:
+	  return code_to_optab (GET_CODE (x));
+	}
+    }
+  if (UNARY_P (x))
+    {
+      if (!noce_can_force_operand (XEXP (x, 0)))
+	return false;
+      switch (GET_CODE (x))
+	{
+	case ZERO_EXTEND:
+	case SIGN_EXTEND:
+	case TRUNCATE:
+	case FLOAT_EXTEND:
+	case FLOAT_TRUNCATE:
+	case FIX:
+	case UNSIGNED_FIX:
+	case FLOAT:
+	case UNSIGNED_FLOAT:
+	  return true;
+	default:
+	  return code_to_optab (GET_CODE (x));
+	}
+    }
+  return false;
+}
+
 /* Emit instruction to move an rtx, possibly into STRICT_LOW_PART.
    X is the destination/target and Y is the value to copy.  */
 
@@ -943,7 +997,7 @@ noce_emit_move_insn (rtx x, rtx y)
 	    {
 	    case RTX_UNARY:
 	      ot = code_to_optab (GET_CODE (y));
-	      if (ot)
+	      if (ot && noce_can_force_operand (XEXP (y, 0)))
 		{
 		  start_sequence ();
 		  target = expand_unop (GET_MODE (y), ot, XEXP (y, 0), x, 0);
@@ -960,7 +1014,9 @@ noce_emit_move_insn (rtx x, rtx y)
 	    case RTX_BIN_ARITH:
 	    case RTX_COMM_ARITH:
 	      ot = code_to_optab (GET_CODE (y));
-	      if (ot)
+	      if (ot
+		  && noce_can_force_operand (XEXP (y, 0))
+		  && noce_can_force_operand (XEXP (y, 1)))
 		{
 		  start_sequence ();
 		  target = expand_binop (GET_MODE (y), ot,
@@ -2763,15 +2819,18 @@ noce_try_sign_mask (struct noce_if_info *if_info)
      INSN_B which can happen for e.g. conditional stores to memory.  For the
      cost computation use the block TEST_BB where the evaluation will end up
      after the transformation.  */
-  t_unconditional =
-    (t == if_info->b
-     && (if_info->insn_b == NULL_RTX
-	 || BLOCK_FOR_INSN (if_info->insn_b) == if_info->test_bb));
+  t_unconditional
+    = (t == if_info->b
+       && (if_info->insn_b == NULL_RTX
+	   || BLOCK_FOR_INSN (if_info->insn_b) == if_info->test_bb));
   if (!(t_unconditional
 	|| (set_src_cost (t, mode, if_info->speed_p)
 	    < COSTS_N_INSNS (2))))
     return FALSE;
 
+  if (!noce_can_force_operand (t))
+    return FALSE;
+
   start_sequence ();
   /* Use emit_store_flag to generate "m < 0 ? -1 : 0" instead of expanding
      "(signed) m >> 31" directly.  This benefits targets with specialized
diff --git a/gcc/testsuite/gcc.dg/pr97487-1.c b/gcc/testsuite/gcc.dg/pr97487-1.c
new file mode 100644
index 0000000..e79d1f1
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr97487-1.c
@@ -0,0 +1,9 @@
+/* PR middle-end/97487 */
+/* { dg-do compile } */
+/* { dg-options "-O2 --param max-rtl-if-conversion-unpredictable-cost=0" } */
+
+long int __attribute__ ((simd))
+foo (long int x, long int y)
+{
+  return x < 0 ? y : 0;
+}
diff --git a/gcc/testsuite/gcc.dg/pr97487-2.c b/gcc/testsuite/gcc.dg/pr97487-2.c
new file mode 100644
index 0000000..0b62381
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr97487-2.c
@@ -0,0 +1,18 @@
+/* PR middle-end/97487 */
+/* { dg-do compile } */
+/* { dg-options "-O2 --param max-rtl-if-conversion-unpredictable-cost=0 -Wno-psabi -w" } */
+
+typedef long long int V __attribute__((vector_size (16)));
+
+long long int
+foo (V x, V y)
+{
+  long long int t1 = y[0];
+  long long int t2 = x[0];
+  long long int t3;
+  if (t2 < 0)
+    t3 = t1;
+  else
+    t3 = 0;
+  return t3;
+}