tree-optimization/101202 - fix ICE with failed backedge SLP nodes

This fixes an ICE with failed backedge SLP nodes still in the graph
while doing permute optimization by explicitely handling those.

2021-06-25  Richard Biener  <rguenther@suse.de>

	PR tree-optimization/101202
	* tree-vect-slp.c (vect_optimize_slp): Explicitely handle
	failed nodes.

	* gcc.dg/torture/pr101202.c: New testcase.

2 files changed, 62 insertions, 20 deletions

diff --git a/gcc/testsuite/gcc.dg/torture/pr101202.c b/gcc/testsuite/gcc.dg/torture/pr101202.c
new file mode 100644
index 0000000..e76c908
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr101202.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-ftree-vectorize" } */
+
+int printf(const char *, ...);
+unsigned a, b, d;
+int c, e, f;
+int main()
+{
+  while (a)
+    if (b)
+      {
+	f = a;
+	while (e)
+	  {
+	    int h, i;
+	    if (d)
+	      {
+		h = a;
+		i = d;
+L:
+		d = a | d && c;
+		if (a)
+		  {
+		    printf("%d", a);
+		    goto L;
+		  }
+	      }
+	    a = h;
+	    d = i;
+	  }
+      }
+  return 0;
+}
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index 227d6aa..17fe5f2 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -3689,26 +3689,33 @@ vect_optimize_slp (vec_info *vinfo)
 
 	  vertices[idx].visited = 1;
 
-	  /* We do not handle stores with a permutation.  */
-	  stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
-	  if (STMT_VINFO_DATA_REF (rep)
-	      && DR_IS_WRITE (STMT_VINFO_DATA_REF (rep)))
-	    continue;
-	  /* We cannot move a permute across an operation that is
-	     not independent on lanes.  Note this is an explicit
-	     negative list since that's much shorter than the respective
-	     positive one but it's critical to keep maintaining it.  */
-	  if (is_gimple_call (STMT_VINFO_STMT (rep)))
-	    switch (gimple_call_combined_fn (STMT_VINFO_STMT (rep)))
-	      {
-	      case CFN_COMPLEX_ADD_ROT90:
-	      case CFN_COMPLEX_ADD_ROT270:
-	      case CFN_COMPLEX_MUL:
-	      case CFN_COMPLEX_MUL_CONJ:
-	      case CFN_VEC_ADDSUB:
+	  /* We still eventually have failed backedge SLP nodes in the
+	     graph, those are only cancelled when analyzing operations.
+	     Simply treat them as transparent ops, propagating permutes
+	     through them.  */
+	  if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
+	    {
+	      /* We do not handle stores with a permutation.  */
+	      stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
+	      if (STMT_VINFO_DATA_REF (rep)
+		  && DR_IS_WRITE (STMT_VINFO_DATA_REF (rep)))
 		continue;
-	      default:;
-	      }
+	      /* We cannot move a permute across an operation that is
+		 not independent on lanes.  Note this is an explicit
+		 negative list since that's much shorter than the respective
+		 positive one but it's critical to keep maintaining it.  */
+	      if (is_gimple_call (STMT_VINFO_STMT (rep)))
+		switch (gimple_call_combined_fn (STMT_VINFO_STMT (rep)))
+		  {
+		  case CFN_COMPLEX_ADD_ROT90:
+		  case CFN_COMPLEX_ADD_ROT270:
+		  case CFN_COMPLEX_MUL:
+		  case CFN_COMPLEX_MUL_CONJ:
+		  case CFN_VEC_ADDSUB:
+		    continue;
+		  default:;
+		  }
+	    }
 
 	  int perm = -1;
 	  for (graph_edge *succ = slpg->vertices[idx].succ;
@@ -3812,7 +3819,9 @@ vect_optimize_slp (vec_info *vinfo)
       slp_tree child;
       FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
 	{
-	  if (!child || SLP_TREE_DEF_TYPE (child) == vect_internal_def)
+	  if (!child
+	      || (SLP_TREE_DEF_TYPE (child) != vect_constant_def
+		  && SLP_TREE_DEF_TYPE (child) != vect_external_def))
 	    continue;
 
 	  /* If the vector is uniform there's nothing to do.  */