Refactor try_vectorize_loop_1

This refactors epilogue loop handling in try_vectorize_loop_1 to not
suggest we're analyzing those there by splitting out the transform
phase which then can handle the epilogues.

2021-10-27  Richard Biener  <rguenther@suse.de>

	* tree-vectorizer.c (vect_transform_loops): New function,
	split out from ...
	(try_vectorize_loop_1): ... here.  Simplify as epilogues
	are now fully handled in the split part.

1 files changed, 50 insertions, 55 deletions

diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c
index 4712dc6..89fa883 100644
--- a/gcc/tree-vectorizer.c
+++ b/gcc/tree-vectorizer.c
@@ -979,6 +979,50 @@ set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
   free (bbs);
 }
 
+/* Generate vectorized code for LOOP and its epilogues.  */
+
+static void
+vect_transform_loops (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
+		      loop_p loop, gimple *loop_vectorized_call)
+{
+  loop_vec_info loop_vinfo = loop_vec_info_for_loop (loop);
+
+  if (loop_vectorized_call)
+    set_uid_loop_bbs (loop_vinfo, loop_vectorized_call);
+
+  unsigned HOST_WIDE_INT bytes;
+  if (dump_enabled_p ())
+    {
+      if (GET_MODE_SIZE (loop_vinfo->vector_mode).is_constant (&bytes))
+	dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
+			 "loop vectorized using %wu byte vectors\n", bytes);
+      else
+	dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
+			 "loop vectorized using variable length vectors\n");
+    }
+
+  loop_p new_loop = vect_transform_loop (loop_vinfo,
+					 loop_vectorized_call);
+  /* Now that the loop has been vectorized, allow it to be unrolled
+     etc.  */
+  loop->force_vectorize = false;
+
+  if (loop->simduid)
+    {
+      simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf);
+      if (!simduid_to_vf_htab)
+	simduid_to_vf_htab = new hash_table<simduid_to_vf> (15);
+      simduid_to_vf_data->simduid = DECL_UID (loop->simduid);
+      simduid_to_vf_data->vf = loop_vinfo->vectorization_factor;
+      *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT)
+	  = simduid_to_vf_data;
+    }
+
+  /* Epilogue of vectorized loop must be vectorized too.  */
+  if (new_loop)
+    vect_transform_loops (simduid_to_vf_htab, new_loop, NULL);
+}
+
 /* Try to vectorize LOOP.  */
 
 static unsigned
@@ -999,17 +1043,9 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
 		 LOCATION_FILE (vect_location.get_location_t ()),
 		 LOCATION_LINE (vect_location.get_location_t ()));
 
-  opt_loop_vec_info loop_vinfo = opt_loop_vec_info::success (NULL);
-  /* In the case of epilogue vectorization the loop already has its
-     loop_vec_info set, we do not require to analyze the loop in this case.  */
-  if (loop_vec_info vinfo = loop_vec_info_for_loop (loop))
-    loop_vinfo = opt_loop_vec_info::success (vinfo);
-  else
-    {
-      /* Try to analyze the loop, retaining an opt_problem if dump_enabled_p.  */
-      loop_vinfo = vect_analyze_loop (loop, &shared);
-      loop->aux = loop_vinfo;
-    }
+  /* Try to analyze the loop, retaining an opt_problem if dump_enabled_p.  */
+  opt_loop_vec_info loop_vinfo = vect_analyze_loop (loop, &shared);
+  loop->aux = loop_vinfo;
 
   if (!loop_vinfo)
     if (dump_enabled_p ())
@@ -1083,8 +1119,7 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
       return ret;
     }
 
-  /* Only count the original scalar loops.  */
-  if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo) && !dbg_cnt (vect_loop))
+  if (!dbg_cnt (vect_loop))
     {
       /* Free existing information if loop is analyzed with some
 	 assumptions.  */
@@ -1093,62 +1128,22 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
       return ret;
     }
 
-  if (loop_vectorized_call)
-    set_uid_loop_bbs (loop_vinfo, loop_vectorized_call);
-
-  unsigned HOST_WIDE_INT bytes;
-  if (dump_enabled_p ())
-    {
-      if (GET_MODE_SIZE (loop_vinfo->vector_mode).is_constant (&bytes))
-	dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
-			 "loop vectorized using %wu byte vectors\n", bytes);
-      else
-	dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
-			 "loop vectorized using variable length vectors\n");
-    }
-
-  loop_p new_loop = vect_transform_loop (loop_vinfo,
-					 loop_vectorized_call);
   (*num_vectorized_loops)++;
-  /* Now that the loop has been vectorized, allow it to be unrolled
-     etc.  */
-  loop->force_vectorize = false;
-
-  if (loop->simduid)
-    {
-      simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf);
-      if (!simduid_to_vf_htab)
-	simduid_to_vf_htab = new hash_table<simduid_to_vf> (15);
-      simduid_to_vf_data->simduid = DECL_UID (loop->simduid);
-      simduid_to_vf_data->vf = loop_vinfo->vectorization_factor;
-      *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT)
-	  = simduid_to_vf_data;
-    }
+  /* Transform LOOP and its epilogues.  */
+  vect_transform_loops (simduid_to_vf_htab, loop, loop_vectorized_call);
 
   if (loop_vectorized_call)
     {
       fold_loop_internal_call (loop_vectorized_call, boolean_true_node);
-      loop_vectorized_call = NULL;
       ret |= TODO_cleanup_cfg;
     }
   if (loop_dist_alias_call)
     {
       tree value = gimple_call_arg (loop_dist_alias_call, 1);
       fold_loop_internal_call (loop_dist_alias_call, value);
-      loop_dist_alias_call = NULL;
       ret |= TODO_cleanup_cfg;
     }
 
-  /* Epilogue of vectorized loop must be vectorized too.  */
-  if (new_loop)
-    {
-      /* Don't include vectorized epilogues in the "vectorized loops" count.
-       */
-      unsigned dont_count = *num_vectorized_loops;
-      ret |= try_vectorize_loop_1 (simduid_to_vf_htab, &dont_count,
-				   new_loop, NULL, NULL);
-    }
-
   return ret;
 }