tree-optimization/97558 - compute vectype for SLP nested cycles

This makes sure to compute the vector type for invariant SLP children
of nested cycles.

2020-11-02  Richard Biener  <rguenther@suse.de>

	PR tree-optimization/97558
	* tree-vect-loop.c (vectorizable_reduction): For nested SLP
	cycles compute invariant operands vector type.

	* gcc.dg/vect/pr97558-2.c: New testcase.

2 files changed, 76 insertions, 3 deletions

diff --git a/gcc/testsuite/gcc.dg/vect/pr97558-2.c b/gcc/testsuite/gcc.dg/vect/pr97558-2.c
new file mode 100644
index 0000000..8f08086
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr97558-2.c
@@ -0,0 +1,54 @@
+/* { dg-do run } */
+/* { dg-additional-options "-O3 -fno-tree-forwprop -fno-tree-scev-cprop" } */
+
+#include "tree-vect.h"
+
+#define N 40
+
+int a[N];
+int b[N];
+
+__attribute__ ((noinline)) int
+foo (int n){
+  int i,j;
+  int sum,x,y;
+
+  if (n<=0)
+    return 0;
+
+  for (i = 0; i < N/2; i++) {
+    sum = 0;
+    x = b[2*i];
+    y = b[2*i+1];
+    for (j = 0; j < n; j++) {
+      sum += j;
+    }
+    a[2*i] = sum + x;
+    a[2*i+1] = sum + y;
+  }
+}
+
+int main (void)
+{
+  int i,j;
+  int sum;
+
+  check_vect ();
+
+  for (i=0; i<N; i++)
+    b[i] = i;
+ 
+  foo (N-1);
+
+    /* check results:  */
+  for (i=0; i<N/2; i++)
+    {
+      sum = 0;
+      for (j = 0; j < N-1; j++)
+        sum += j;
+      if (a[2*i] != sum + b[2*i] || a[2*i+1] != sum + b[2*i+1])
+        abort();
+    }
+
+  return 0;
+}
diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index 353703c..105ea61 100644
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -6336,9 +6336,28 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
   if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
     {
       if (is_a <gphi *> (stmt_info->stmt))
-	/* Analysis for double-reduction is done on the outer
-	   loop PHI, nested cycles have no further restrictions.  */
-	STMT_VINFO_TYPE (stmt_info) = cycle_phi_info_type;
+	{
+	  if (slp_node)
+	    {
+	      /* We eventually need to set a vector type on invariant
+		 arguments.  */
+	      unsigned j;
+	      slp_tree child;
+	      FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
+		if (!vect_maybe_update_slp_op_vectype
+		       (child, SLP_TREE_VECTYPE (slp_node)))
+		  {
+		    if (dump_enabled_p ())
+		      dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+				       "incompatible vector types for "
+				       "invariants\n");
+		    return false;
+		  }
+	    }
+	  /* Analysis for double-reduction is done on the outer
+	     loop PHI, nested cycles have no further restrictions.  */
+	  STMT_VINFO_TYPE (stmt_info) = cycle_phi_info_type;
+	}
       else
 	STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
       return true;