tree-optimization/96028 - fix bogus externalizing of SLP node

This guards externalizing a SLP node when it fails to code generate
to actually have scalar defs we can use.  It also makes failure
to do so not fell the whole SLP instance but instead try this again
on the parent.

2020-07-02  Richard Biener  <rguenther@suse.de>

	PR tree-optimization/96028
	* tree-vect-slp.c (vect_slp_convert_to_external): Make sure
	we have scalar stmts to use.
	(vect_slp_analyze_node_operations): When analyzing a child
	failed try externalizing the parent node.

1 files changed, 18 insertions, 9 deletions

diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index af123b5..eff68f7 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -2789,6 +2789,7 @@ vect_slp_convert_to_external (vec_info *vinfo, slp_tree node,
 
   if (!is_a <bb_vec_info> (vinfo)
       || node == SLP_INSTANCE_TREE (node_instance)
+      || !SLP_TREE_SCALAR_STMTS (node).exists ()
       || vect_contains_pattern_stmt_p (SLP_TREE_SCALAR_STMTS (node)))
     return false;
 
@@ -2893,16 +2894,25 @@ vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
      doesn't result in any issue since we throw away the lvisited set
      when we fail.  */
   if (visited.contains (node)
-      || lvisited.add (node))
+      || lvisited.contains (node))
     return true;
 
+  bool res = true;
   FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
-    if (!vect_slp_analyze_node_operations (vinfo, child, node_instance,
-					   visited, lvisited, cost_vec))
-      return false;
+    {
+      res = vect_slp_analyze_node_operations (vinfo, child, node_instance,
+					      visited, lvisited, cost_vec);
+      if (!res)
+	break;
+    }
 
-  bool res = vect_slp_analyze_node_operations_1 (vinfo, node, node_instance,
-						 cost_vec);
+  if (res)
+    {
+      res = vect_slp_analyze_node_operations_1 (vinfo, node, node_instance,
+						cost_vec);
+      if (res)
+	lvisited.add (node);
+    }
 
   /* When the node can be vectorized cost invariant nodes it references.
      This is not done in DFS order to allow the refering node
@@ -2944,13 +2954,12 @@ vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
 	  vect_prologue_cost_for_slp (child, cost_vec);
 	}
 
-  /* If this node can't be vectorized, try pruning the tree here rather
-     than felling the whole thing.  */
+  /* If this node or any of its children can't be vectorized, try pruning
+     the tree here rather than felling the whole thing.  */
   if (!res && vect_slp_convert_to_external (vinfo, node, node_instance))
     {
       /* We'll need to revisit this for invariant costing and number
 	 of vectorized stmt setting.   */
-      lvisited.remove (node);
       res = true;
     }