tree-vectorizer.h (struct _bb_vec_info): Add region_begin/end members.

2015-11-06  Richard Biener  <rguenther@suse.de>

	* tree-vectorizer.h (struct _bb_vec_info): Add region_begin/end
	members.
	(vect_stmt_in_region_p): Declare.
	* tree-vect-slp.c (new_bb_vec_info): Work on a region.
	(destroy_bb_vec_info): Likewise.
	(vect_bb_slp_scalar_cost): Use vect_stmt_in_region_p.
	(vect_get_and_check_slp_defs): Likewise.
	(vect_slp_analyze_bb_1): Refactor to make it work on sub-BBs.
	(vect_slp_bb): Likewise.
	* tree-vect-patterns.c (vect_same_loop_or_bb_p): Implement
	in terms of vect_stmt_in_region_p.
	(vect_pattern_recog): Iterate over the BB region.
	* tree-vect-stmts.c (vect_is_simple_use): Use vect_stmt_in_region_p.
	* tree-vectorizer.c (vect_stmt_in_region_p): New function.
	(pass_slp_vectorize::execute): Initialize all stmt UIDs to -1.

	* config/i386/i386.c: Include gimple-iterator.h.
	* config/aarch64/aarch64.c: Likewise.

	* gcc.dg/vect/bb-slp-38.c: New testcase.

From-SVN: r229842

10 files changed, 240 insertions, 128 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index ae2c5cf..0355c9e 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,23 @@
+2015-11-06  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (struct _bb_vec_info): Add region_begin/end
+	members.
+	(vect_stmt_in_region_p): Declare.
+	* tree-vect-slp.c (new_bb_vec_info): Work on a region.
+	(destroy_bb_vec_info): Likewise.
+	(vect_bb_slp_scalar_cost): Use vect_stmt_in_region_p.
+	(vect_get_and_check_slp_defs): Likewise.
+	(vect_slp_analyze_bb_1): Refactor to make it work on sub-BBs.
+	(vect_slp_bb): Likewise.
+	* tree-vect-patterns.c (vect_same_loop_or_bb_p): Implement
+	in terms of vect_stmt_in_region_p.
+	(vect_pattern_recog): Iterate over the BB region.
+	* tree-vect-stmts.c (vect_is_simple_use): Use vect_stmt_in_region_p.
+	* tree-vectorizer.c (vect_stmt_in_region_p): New function.
+	(pass_slp_vectorize::execute): Initialize all stmt UIDs to -1.
+	* config/i386/i386.c: Include gimple-iterator.h.
+	* config/aarch64/aarch64.c: Likewise.
+
 2015-11-06  Alexandre Oliva <aoliva@redhat.com>
 
 	PR rtl-optimization/67753
diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c
index 5c8604f..98b13b4 100644
--- a/gcc/config/aarch64/aarch64.c
+++ b/gcc/config/aarch64/aarch64.c
@@ -52,6 +52,7 @@
 #include "params.h"
 #include "gimplify.h"
 #include "dwarf2.h"
+#include "gimple-iterator.h"
 #include "tree-vectorizer.h"
 #include "aarch64-cost-tables.h"
 #include "dumpfile.h"
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index 2a965f6..bb37aba 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -64,6 +64,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "context.h"
 #include "pass_manager.h"
 #include "target-globals.h"
+#include "gimple-iterator.h"
 #include "tree-vectorizer.h"
 #include "shrink-wrap.h"
 #include "builtins.h"
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index b0e78e9..3d1834d 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,7 @@
+2015-11-06  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/vect/bb-slp-38.c: New testcase.
+
 2015-11-06  Thomas Schwinge  <thomas@codesourcery.com>
 
 	* gfortran.dg/goacc/private-3.f95: XFAIL.
diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-38.c b/gcc/testsuite/gcc.dg/vect/bb-slp-38.c
new file mode 100644
index 0000000..59aec54
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-38.c
@@ -0,0 +1,44 @@
+/* { dg-require-effective-target vect_int } */
+
+#include "tree-vect.h"
+
+extern void abort (void);
+
+int a[8], b[8];
+int x;
+
+void __attribute__((noinline,noclone))
+bar (void)
+{
+  x = 1;
+}
+
+void __attribute__((noinline,noclone))
+foo(void)
+{
+  a[0] = b[0];
+  a[1] = b[0];
+  a[2] = b[3];
+  a[3] = b[3];
+  bar ();
+  a[4] = b[4];
+  a[5] = b[7];
+  a[6] = b[4];
+  a[7] = b[7];
+}
+
+int main()
+{
+  int i;
+  check_vect ();
+  for (i = 0; i < 8; ++i)
+    b[i] = i;
+  foo ();
+  if (a[0] != 0 || a[1] != 0 || a[2] != 3 || a[3] != 3
+      || a[4] != 4 || a[5] != 7 || a[6] != 4 || a[7] != 7)
+    abort ();
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "basic block vectorized" "slp2" { target vect_perm } } } */
+/* { dg-final { scan-tree-dump-times "basic block part vectorized" 2 "slp2" { target vect_perm } } } */
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
index f1de690..d003d33 100644
--- a/gcc/tree-vect-patterns.c
+++ b/gcc/tree-vect-patterns.c
@@ -107,27 +107,7 @@ static bool
 vect_same_loop_or_bb_p (gimple *stmt1, gimple *stmt2)
 {
   stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt1);
-  loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
-  bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
-
-  if (!gimple_bb (stmt2))
-    return false;
-
-  if (loop_vinfo)
-    {
-      struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-      if (!flow_bb_inside_loop_p (loop, gimple_bb (stmt2)))
-	return false;
-    }
-  else
-    {
-      if (gimple_bb (stmt2) != BB_VINFO_BB (bb_vinfo)
-	  || gimple_code (stmt2) == GIMPLE_PHI)
-	return false;
-    }
-
-  gcc_assert (vinfo_for_stmt (stmt2));
-  return true;
+  return vect_stmt_in_region_p (stmt_vinfo->vinfo, stmt2);
 }
 
 /* If the LHS of DEF_STMT has a single use, and that statement is
@@ -3611,33 +3591,42 @@ vect_pattern_recog (vec_info *vinfo)
       loop = LOOP_VINFO_LOOP (loop_vinfo);
       bbs = LOOP_VINFO_BBS (loop_vinfo);
       nbbs = loop->num_nodes;
+
+      /* Scan through the loop stmts, applying the pattern recognition
+	 functions starting at each stmt visited:  */
+      for (i = 0; i < nbbs; i++)
+	{
+	  basic_block bb = bbs[i];
+	  for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
+	    {
+	      /* Scan over all generic vect_recog_xxx_pattern functions.  */
+	      for (j = 0; j < NUM_PATTERNS; j++)
+		{
+		  vect_recog_func = vect_vect_recog_func_ptrs[j];
+		  vect_pattern_recog_1 (vect_recog_func, si,
+					&stmts_to_replace);
+		}
+	    }
+	}
     }
   else
     {
-      bbs = &as_a <bb_vec_info> (vinfo)->bb;
-      nbbs = 1;
-    }
-
-  /* Scan through the loop stmts, applying the pattern recognition
-     functions starting at each stmt visited:  */
-  for (i = 0; i < nbbs; i++)
-    {
-      basic_block bb = bbs[i];
-      for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
-        {
-	  if (is_a <bb_vec_info> (vinfo)
-	      && (stmt = gsi_stmt (si))
+      bb_vec_info bb_vinfo = as_a <bb_vec_info> (vinfo);
+      for (si = bb_vinfo->region_begin;
+	   gsi_stmt (si) != gsi_stmt (bb_vinfo->region_end); gsi_next (&si))
+	{
+	  if ((stmt = gsi_stmt (si))
 	      && vinfo_for_stmt (stmt)
 	      && !STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)))
-	   continue;
+	    continue;
 
-          /* Scan over all generic vect_recog_xxx_pattern functions.  */
-          for (j = 0; j < NUM_PATTERNS; j++)
-            {
+	  /* Scan over all generic vect_recog_xxx_pattern functions.  */
+	  for (j = 0; j < NUM_PATTERNS; j++)
+	    {
 	      vect_recog_func = vect_vect_recog_func_ptrs[j];
 	      vect_pattern_recog_1 (vect_recog_func, si,
 				    &stmts_to_replace);
-            }
-        }
+	    }
+	}
     }
 }
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index 35808eb..cfdfc29 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -209,7 +209,6 @@ vect_get_and_check_slp_defs (vec_info *vinfo,
   unsigned int i, number_of_oprnds;
   gimple *def_stmt;
   enum vect_def_type dt = vect_uninitialized_def;
-  struct loop *loop = NULL;
   bool pattern = false;
   slp_oprnd_info oprnd_info;
   int first_op_idx = 1;
@@ -218,9 +217,6 @@ vect_get_and_check_slp_defs (vec_info *vinfo,
   bool first = stmt_num == 0;
   bool second = stmt_num == 1;
 
-  if (is_a <loop_vec_info> (vinfo))
-    loop = LOOP_VINFO_LOOP (as_a <loop_vec_info> (vinfo));
-
   if (is_gimple_call (stmt))
     {
       number_of_oprnds = gimple_call_num_args (stmt);
@@ -276,11 +272,7 @@ again:
          from the pattern.  Check that all the stmts of the node are in the
          pattern.  */
       if (def_stmt && gimple_bb (def_stmt)
-          && ((is_a <loop_vec_info> (vinfo)
-	       && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
-	      || (is_a <bb_vec_info> (vinfo)
-		  && gimple_bb (def_stmt) == as_a <bb_vec_info> (vinfo)->bb
-		  && gimple_code (def_stmt) != GIMPLE_PHI))
+          && vect_stmt_in_region_p (vinfo, def_stmt)
           && vinfo_for_stmt (def_stmt)
           && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (def_stmt))
 	  && !STMT_VINFO_RELEVANT (vinfo_for_stmt (def_stmt))
@@ -2076,16 +2068,21 @@ vect_detect_hybrid_slp (loop_vec_info loop_vinfo)
    stmt_vec_info structs for all the stmts in it.  */
 
 static bb_vec_info
-new_bb_vec_info (basic_block bb)
+new_bb_vec_info (gimple_stmt_iterator region_begin,
+		 gimple_stmt_iterator region_end)
 {
+  basic_block bb = gsi_bb (region_begin);
   bb_vec_info res = NULL;
   gimple_stmt_iterator gsi;
 
   res = (bb_vec_info) xcalloc (1, sizeof (struct _bb_vec_info));
   res->kind = vec_info::bb;
   BB_VINFO_BB (res) = bb;
+  res->region_begin = region_begin;
+  res->region_end = region_end;
 
-  for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+  for (gsi = region_begin; gsi_stmt (gsi) != gsi_stmt (region_end);
+       gsi_next (&gsi))
     {
       gimple *stmt = gsi_stmt (gsi);
       gimple_set_uid (stmt, 0);
@@ -2118,7 +2115,8 @@ destroy_bb_vec_info (bb_vec_info bb_vinfo)
 
   bb = BB_VINFO_BB (bb_vinfo);
 
-  for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
+  for (si = bb_vinfo->region_begin;
+       gsi_stmt (si) != gsi_stmt (bb_vinfo->region_end); gsi_next (&si))
     {
       gimple *stmt = gsi_stmt (si);
       stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
@@ -2126,6 +2124,9 @@ destroy_bb_vec_info (bb_vec_info bb_vinfo)
       if (stmt_info)
         /* Free stmt_vec_info.  */
         free_stmt_vec_info (stmt);
+
+      /* Reset region marker.  */
+      gimple_set_uid (stmt, -1);
     }
 
   vect_destroy_datarefs (bb_vinfo);
@@ -2247,8 +2248,8 @@ vect_bb_slp_scalar_cost (basic_block bb,
 	  gimple *use_stmt;
 	  FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, DEF_FROM_PTR (def_p))
 	    if (!is_gimple_debug (use_stmt)
-		&& (gimple_code (use_stmt) == GIMPLE_PHI
-		    || gimple_bb (use_stmt) != bb
+		&& (! vect_stmt_in_region_p (vinfo_for_stmt (stmt)->vinfo,
+					     use_stmt)
 		    || !STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (use_stmt))))
 	      {
 		(*life)[i] = true;
@@ -2327,40 +2328,31 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo)
 /* Check if the basic block can be vectorized.  */
 
 static bb_vec_info
-vect_slp_analyze_bb_1 (basic_block bb)
+vect_slp_analyze_bb_1 (gimple_stmt_iterator region_begin,
+		       gimple_stmt_iterator region_end,
+		       vec<data_reference_p> datarefs, int n_stmts)
 {
   bb_vec_info bb_vinfo;
   vec<slp_instance> slp_instances;
   slp_instance instance;
   int i;
   int min_vf = 2;
-  unsigned n_stmts = 0;
 
-  bb_vinfo = new_bb_vec_info (bb);
+  if (n_stmts > PARAM_VALUE (PARAM_SLP_MAX_INSNS_IN_BB))
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "not vectorized: too many instructions in "
+			 "basic block.\n");
+      free_data_refs (datarefs);
+      return NULL;
+    }
+
+  bb_vinfo = new_bb_vec_info (region_begin, region_end);
   if (!bb_vinfo)
     return NULL;
 
-  /* Gather all data references in the basic-block.  */
-
-  for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
-       !gsi_end_p (gsi); gsi_next (&gsi))
-    {
-      gimple *stmt = gsi_stmt (gsi);
-      if (is_gimple_debug (stmt))
-	continue;
-      ++n_stmts;
-      if (!find_data_references_in_stmt (NULL, stmt,
-					 &BB_VINFO_DATAREFS (bb_vinfo)))
-	{
-	  /* Mark the rest of the basic-block as unvectorizable.  */
-	  for (; !gsi_end_p (gsi); gsi_next (&gsi))
-	    {
-	      stmt = gsi_stmt (gsi);
-	      STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)) = false;
-	    }
-	  break;
-	}
-    }
+  BB_VINFO_DATAREFS (bb_vinfo) = datarefs;
 
   /* Analyze the data references.  */
 
@@ -2438,8 +2430,8 @@ vect_slp_analyze_bb_1 (basic_block bb)
     }
 
   /* Mark all the statements that we do not want to vectorize.  */
-  for (gimple_stmt_iterator gsi = gsi_start_bb (BB_VINFO_BB (bb_vinfo));
-       !gsi_end_p (gsi); gsi_next (&gsi))
+  for (gimple_stmt_iterator gsi = bb_vinfo->region_begin;
+       gsi_stmt (gsi) != gsi_stmt (bb_vinfo->region_end); gsi_next (&gsi))
     {
       stmt_vec_info vinfo = vinfo_for_stmt (gsi_stmt (gsi));
       if (STMT_SLP_TYPE (vinfo) != pure_slp)
@@ -2509,77 +2501,105 @@ bool
 vect_slp_bb (basic_block bb)
 {
   bb_vec_info bb_vinfo;
-  int insns = 0;
   gimple_stmt_iterator gsi;
   unsigned int vector_sizes;
+  bool any_vectorized = false;
 
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location, "===vect_slp_analyze_bb===\n");
 
-  for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
-    {
-      gimple *stmt = gsi_stmt (gsi);
-      if (!is_gimple_debug (stmt)
-          && !gimple_nop_p (stmt)
-          && gimple_code (stmt) != GIMPLE_LABEL)
-        insns++;
-      if (gimple_location (stmt) != UNKNOWN_LOCATION)
-	vect_location = gimple_location (stmt);
-    }
-
-  if (insns > PARAM_VALUE (PARAM_SLP_MAX_INSNS_IN_BB))
-    {
-      if (dump_enabled_p ())
-        dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			 "not vectorized: too many instructions in "
-			 "basic block.\n");
-
-      return false;
-    }
-
   /* Autodetect first vector size we try.  */
   current_vector_size = 0;
   vector_sizes = targetm.vectorize.autovectorize_vector_sizes ();
 
+  gsi = gsi_start_bb (bb);
+
   while (1)
     {
-      bb_vinfo = vect_slp_analyze_bb_1 (bb);
-      if (bb_vinfo)
+      if (gsi_end_p (gsi))
+	break;
+
+      gimple_stmt_iterator region_begin = gsi;
+      vec<data_reference_p> datarefs = vNULL;
+      int insns = 0;
+
+      for (; !gsi_end_p (gsi); gsi_next (&gsi))
 	{
-	  if (!dbg_cnt (vect_slp))
-	    {
-	      destroy_bb_vec_info (bb_vinfo);
-	      return false;
-	    }
+	  gimple *stmt = gsi_stmt (gsi);
+	  if (is_gimple_debug (stmt))
+	    continue;
+	  insns++;
+
+	  if (gimple_location (stmt) != UNKNOWN_LOCATION)
+	    vect_location = gimple_location (stmt);
+
+	  if (!find_data_references_in_stmt (NULL, stmt, &datarefs))
+	    break;
+	}
+
+      /* Skip leading unhandled stmts.  */
+      if (gsi_stmt (region_begin) == gsi_stmt (gsi))
+	{
+	  gsi_next (&gsi);
+	  continue;
+	}
 
+      gimple_stmt_iterator region_end = gsi;
+
+      bool vectorized = false;
+      bb_vinfo = vect_slp_analyze_bb_1 (region_begin, region_end,
+					datarefs, insns);
+      if (bb_vinfo
+	  && dbg_cnt (vect_slp))
+	{
 	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB\n");
+	    dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB part\n");
 
 	  vect_schedule_slp (bb_vinfo);
 
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_NOTE, vect_location,
-			     "BASIC BLOCK VECTORIZED\n");
+			     "basic block part vectorized\n");
 
 	  destroy_bb_vec_info (bb_vinfo);
 
-	  return true;
+	  vectorized = true;
 	}
+      else
+	destroy_bb_vec_info (bb_vinfo);
 
-      destroy_bb_vec_info (bb_vinfo);
+      any_vectorized |= vectorized;
 
       vector_sizes &= ~current_vector_size;
-      if (vector_sizes == 0
-          || current_vector_size == 0)
-        return false;
+      if (vectorized
+	  || vector_sizes == 0
+	  || current_vector_size == 0)
+	{
+	  if (gsi_end_p (region_end))
+	    break;
 
-      /* Try the next biggest vector size.  */
-      current_vector_size = 1 << floor_log2 (vector_sizes);
-      if (dump_enabled_p ())
-        dump_printf_loc (MSG_NOTE, vect_location,
-			 "***** Re-trying analysis with "
-			 "vector size %d\n", current_vector_size);
+	  /* Skip the unhandled stmt.  */
+	  gsi_next (&gsi);
+
+	  /* And reset vector sizes.  */
+	  current_vector_size = 0;
+	  vector_sizes = targetm.vectorize.autovectorize_vector_sizes ();
+	}
+      else
+	{
+	  /* Try the next biggest vector size.  */
+	  current_vector_size = 1 << floor_log2 (vector_sizes);
+	  if (dump_enabled_p ())
+	    dump_printf_loc (MSG_NOTE, vect_location,
+			     "***** Re-trying analysis with "
+			     "vector size %d\n", current_vector_size);
+
+	  /* Start over.  */
+	  gsi = region_begin;
+	}
     }
+
+  return any_vectorized;
 }
 
 
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index 378973b..96074fc 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -8141,12 +8141,7 @@ vect_is_simple_use (tree operand, vec_info *vinfo,
       dump_gimple_stmt (MSG_NOTE, TDF_SLIM, *def_stmt, 0);
     }
 
-  basic_block bb = gimple_bb (*def_stmt);
-  if ((is_a <loop_vec_info> (vinfo)
-       && !flow_bb_inside_loop_p (as_a <loop_vec_info> (vinfo)->loop, bb))
-      || (is_a <bb_vec_info> (vinfo)
-	  && (bb != as_a <bb_vec_info> (vinfo)->bb
-	      || gimple_code (*def_stmt) == GIMPLE_PHI)))
+  if (! vect_stmt_in_region_p (vinfo, *def_stmt))
     *dt = vect_external_def;
   else
     {
diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c
index 7b3d9a3..41e87a8 100644
--- a/gcc/tree-vectorizer.c
+++ b/gcc/tree-vectorizer.c
@@ -350,6 +350,33 @@ vect_destroy_datarefs (vec_info *vinfo)
 }
 
 
+/* Return whether STMT is inside the region we try to vectorize.  */
+
+bool
+vect_stmt_in_region_p (vec_info *vinfo, gimple *stmt)
+{
+  if (!gimple_bb (stmt))
+    return false;
+
+  if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
+    {
+      struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+      if (!flow_bb_inside_loop_p (loop, gimple_bb (stmt)))
+	return false;
+    }
+  else
+    {
+      bb_vec_info bb_vinfo = as_a <bb_vec_info> (vinfo);
+      if (gimple_bb (stmt) != BB_VINFO_BB (bb_vinfo)
+	  || gimple_uid (stmt) == -1U
+	  || gimple_code (stmt) == GIMPLE_PHI)
+	return false;
+    }
+
+  return true;
+}
+
+
 /* If LOOP has been versioned during ifcvt, return the internal call
    guarding it.  */
 
@@ -692,6 +719,14 @@ pass_slp_vectorize::execute (function *fun)
       scev_initialize ();
     }
 
+  /* Mark all stmts as not belonging to the current region.  */
+  FOR_EACH_BB_FN (bb, fun)
+    {
+      for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
+	   gsi_next (&gsi))
+	gimple_set_uid (gsi_stmt (gsi), -1);
+    }
+
   init_stmt_vec_info_vec ();
 
   FOR_EACH_BB_FN (bb, fun)
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index f77a4eb..9cde091 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -390,6 +390,8 @@ nested_in_vect_loop_p (struct loop *loop, gimple *stmt)
 typedef struct _bb_vec_info : public vec_info
 {
   basic_block bb;
+  gimple_stmt_iterator region_begin;
+  gimple_stmt_iterator region_end;
 } *bb_vec_info;
 
 #define BB_VINFO_BB(B)               (B)->bb
@@ -1085,5 +1087,6 @@ void vect_pattern_recog (vec_info *);
 /* In tree-vectorizer.c.  */
 unsigned vectorize_loops (void);
 void vect_destroy_datarefs (vec_info *);
+bool vect_stmt_in_region_p (vec_info *, gimple *);
 
 #endif  /* GCC_TREE_VECTORIZER_H  */