tree-vectorizer.h (vect_pattern_recog): Add new argument.

ChangeLog:

	* tree-vectorizer.h (vect_pattern_recog): Add new argument.
	* tree-vect-loop.c (vect_analyze_loop_2): Update call to
	vect_pattern_recog.
	* tree-vect-patterns.c (widened_name_p): Pass basic block
	info to vect_is_simple_use.
	(vect_recog_dot_prod_pattern): Fail for basic blocks.
	(vect_recog_widen_sum_pattern): Likewise.
	(vect_handle_widen_op_by_const): Support basic blocks.
	(vect_operation_fits_smaller_type,
	vect_recog_over_widening_pattern): Likewise.
	(vect_recog_vector_vector_shift_pattern): Support basic blocks.
	Update call to vect_is_simple_use.
	(vect_recog_mixed_size_cond_pattern): Support basic blocks.
	Add printing.
	(check_bool_pattern): Add an argument, update call to
	vect_is_simple_use and the recursive calls.
	(vect_recog_bool_pattern): Update relevant function calls.
	Add printing.
	(vect_mark_pattern_stmts): Update calls to new_stmt_vec_info.
	(vect_pattern_recog_1): Check for reduction only in loops.
	(vect_pattern_recog): Add new argument.  Support basic blocks.
	* tree-vect-stmts.c (vectorizable_conversion): Pass basic block
	info to vect_is_simple_use_1.
	* tree-vect-slp.c (vect_get_and_check_slp_defs): Support basic
	blocks.
	(vect_slp_analyze_bb_1): Call vect_pattern_recog.

testsuite/ChangeLog:

	* gcc.dg/vect/bb-slp-pattern-1.c: New test.
	* gcc.dg/vect/bb-slp-pattern-2.c: New test.

Co-Authored-By: Ulrich Weigand <ulrich.weigand@linaro.org>

From-SVN: r185436

9 files changed, 266 insertions, 43 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 1217833..0475186 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,33 @@
+2012-03-15  Ira Rosen  <irar@il.ibm.com>
+	    Ulrich Weigand  <ulrich.weigand@linaro.org>
+
+	* tree-vectorizer.h (vect_pattern_recog): Add new argument.
+	* tree-vect-loop.c (vect_analyze_loop_2): Update call to
+	vect_pattern_recog.
+	* tree-vect-patterns.c (widened_name_p): Pass basic block
+	info to vect_is_simple_use.
+	(vect_recog_dot_prod_pattern): Fail for basic blocks.
+	(vect_recog_widen_sum_pattern): Likewise.
+	(vect_handle_widen_op_by_const): Support basic blocks.
+	(vect_operation_fits_smaller_type,
+	vect_recog_over_widening_pattern): Likewise.
+	(vect_recog_vector_vector_shift_pattern): Support basic blocks.
+	Update call to vect_is_simple_use.
+	(vect_recog_mixed_size_cond_pattern): Support basic blocks.
+	Add printing.
+	(check_bool_pattern): Add an argument, update call to
+	vect_is_simple_use and the recursive calls.
+	(vect_recog_bool_pattern): Update relevant function calls.
+	Add printing.
+	(vect_mark_pattern_stmts): Update calls to new_stmt_vec_info.
+	(vect_pattern_recog_1): Check for reduction only in loops.
+	(vect_pattern_recog): Add new argument.  Support basic blocks.
+	* tree-vect-stmts.c (vectorizable_conversion): Pass basic block
+	info to vect_is_simple_use_1.
+	* tree-vect-slp.c (vect_get_and_check_slp_defs): Support basic
+	blocks.
+	(vect_slp_analyze_bb_1): Call vect_pattern_recog.
+
 2012-03-15  Jakub Jelinek  <jakub@redhat.com>
 	    Andrew Pinski  <apinski@cavium.com>
 
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index e7cce39..dd80387 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,9 @@
+2012-03-15  Ira Rosen  <irar@il.ibm.com>
+	    Ulrich Weigand  <ulrich.weigand@linaro.org>
+
+	* gcc.dg/vect/bb-slp-pattern-1.c: New test.
+	* gcc.dg/vect/bb-slp-pattern-2.c: New test.
+
 2012-03-15  Janne Blomqvist  <jb@gcc.gnu.org>
 
         PR libfortran/52434
diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-pattern-1.c b/gcc/testsuite/gcc.dg/vect/bb-slp-pattern-1.c
new file mode 100644
index 0000000..ff520c0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-pattern-1.c
@@ -0,0 +1,54 @@
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 8
+
+unsigned short X[N];
+unsigned short Y[N];
+unsigned int result[N];
+
+/* unsigned short->unsigned int widening-mult.  */
+__attribute__ ((noinline, noclone)) void
+foo (void)
+{
+  result[0] = (unsigned int) (X[0] * Y[0]);
+  result[1] = (unsigned int) (X[1] * Y[1]);
+  result[2] = (unsigned int) (X[2] * Y[2]);
+  result[3] = (unsigned int) (X[3] * Y[3]);
+  result[4] = (unsigned int) (X[4] * Y[4]);
+  result[5] = (unsigned int) (X[5] * Y[5]);
+  result[6] = (unsigned int) (X[6] * Y[6]);
+  result[7] = (unsigned int) (X[7] * Y[7]);
+}
+
+int main (void)
+{
+  int i, tmp;
+
+  check_vect ();
+
+  for (i = 0; i < N; i++)
+    {
+      X[i] = i;
+      Y[i] = 64-i;
+    }
+
+  foo ();
+
+  for (i = 0; i < N; i++)
+    {
+      __asm__ volatile ("");
+      tmp = X[i] * Y[i];
+      if (result[i] != tmp)
+        abort ();
+    }
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 1 "slp" { target { vect_widen_mult_hi_to_si || vect_unpack } } } } */
+/* { dg-final { scan-tree-dump-times "vect_recog_widen_mult_pattern: detected" 8 "slp" { target vect_widen_mult_hi_to_si_pattern } } } */
+/* { dg-final { scan-tree-dump-times "pattern recognized" 8 "slp" { target vect_widen_mult_hi_to_si_pattern } } } */
+/* { dg-final { cleanup-tree-dump "slp" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-pattern-2.c b/gcc/testsuite/gcc.dg/vect/bb-slp-pattern-2.c
new file mode 100644
index 0000000..30fa487
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-pattern-2.c
@@ -0,0 +1,52 @@
+/* { dg-require-effective-target vect_condition } */
+
+#include "tree-vect.h"
+
+#define N 128
+
+__attribute__((noinline, noclone)) void
+foo (short * __restrict__ a, int * __restrict__ b, int stride)
+{
+  int i;
+
+  for (i = 0; i < N/stride; i++, a += stride, b += stride)
+   {
+     a[0] = b[0] ? 1 : 7;
+     a[1] = b[1] ? 2 : 0;
+     a[2] = b[2] ? 3 : 0;
+     a[3] = b[3] ? 4 : 0;
+     a[4] = b[4] ? 5 : 0;
+     a[5] = b[5] ? 6 : 0;
+     a[6] = b[6] ? 7 : 0;
+     a[7] = b[7] ? 8 : 0;
+   }
+}
+
+short a[N];
+int b[N];
+int main ()
+{
+  int i;
+
+  check_vect ();
+
+  for (i = 0; i < N; i++)
+    {
+      a[i] = i;
+      b[i] = -i;
+    }
+
+  foo (a, b, 8);
+
+  for (i = 1; i < N; i++)
+    if (a[i] != i%8 + 1)
+      abort ();
+
+  if (a[0] != 7)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "basic block vectorized using SLP" 1 "slp" { target { vect_element_align && vect_pack_trunc } } } } */
+/* { dg-final { cleanup-tree-dump "slp" } } */
diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index 7f42a7d..5733bc8 100644
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -1514,7 +1514,7 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo)
 
   vect_analyze_scalar_cycles (loop_vinfo);
 
-  vect_pattern_recog (loop_vinfo);
+  vect_pattern_recog (loop_vinfo, NULL);
 
   /* Data-flow analysis to detect stmts that do not need to be vectorized.  */
 
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
index 2190b8a..8772c0a 100644
--- a/gcc/tree-vect-patterns.c
+++ b/gcc/tree-vect-patterns.c
@@ -105,12 +105,13 @@ widened_name_p (tree name, gimple use_stmt, tree *half_type, gimple *def_stmt,
   tree oprnd0;
   enum vect_def_type dt;
   tree def;
+  bb_vec_info bb_vinfo;
 
   stmt_vinfo = vinfo_for_stmt (use_stmt);
   loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
-
-  if (!vect_is_simple_use (name, use_stmt, loop_vinfo, NULL, def_stmt, &def,
-			   &dt))
+  bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
+  if (!vect_is_simple_use (name, use_stmt, loop_vinfo, bb_vinfo, def_stmt,
+			   &def, &dt))
     return false;
 
   if (dt != vect_internal_def
@@ -135,7 +136,7 @@ widened_name_p (tree name, gimple use_stmt, tree *half_type, gimple *def_stmt,
     return false;
 
   if (!vect_is_simple_use (oprnd0, *def_stmt, loop_vinfo,
-			   NULL, &dummy_gimple, &dummy, &dt))
+			   bb_vinfo, &dummy_gimple, &dummy, &dt))
     return false;
 
   return true;
@@ -211,9 +212,14 @@ vect_recog_dot_prod_pattern (VEC (gimple, heap) **stmts, tree *type_in,
   gimple pattern_stmt;
   tree prod_type;
   loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
-  struct loop *loop = LOOP_VINFO_LOOP (loop_info);
+  struct loop *loop;
   tree var;
 
+  if (!loop_info)
+    return NULL;
+
+  loop = LOOP_VINFO_LOOP (loop_info);
+
   if (!is_gimple_assign (last_stmt))
     return NULL;
 
@@ -383,8 +389,16 @@ vect_handle_widen_op_by_const (gimple stmt, enum tree_code code,
 {
   tree new_type, new_oprnd, tmp;
   gimple new_stmt;
-  loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (vinfo_for_stmt (stmt));
-  struct loop *loop = LOOP_VINFO_LOOP (loop_info);
+  loop_vec_info loop_vinfo;
+  struct loop *loop = NULL;
+  bb_vec_info bb_vinfo;
+  stmt_vec_info stmt_vinfo;
+
+  stmt_vinfo = vinfo_for_stmt (stmt);
+  loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+  bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
+  if (loop_vinfo)
+    loop = LOOP_VINFO_LOOP (loop_vinfo);
 
   if (code != MULT_EXPR && code != LSHIFT_EXPR)
     return false;
@@ -402,7 +416,9 @@ vect_handle_widen_op_by_const (gimple stmt, enum tree_code code,
 
   if (TYPE_PRECISION (type) < (TYPE_PRECISION (*half_type) * 4)
       || !gimple_bb (def_stmt)
-      || !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
+      || (loop && !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
+      || (!loop && gimple_bb (def_stmt) != BB_VINFO_BB (bb_vinfo)
+	  && gimple_code (def_stmt) != GIMPLE_PHI)
       || !vinfo_for_stmt (def_stmt))
     return false;
 
@@ -798,9 +814,14 @@ vect_recog_widen_sum_pattern (VEC (gimple, heap) **stmts, tree *type_in,
   tree type, half_type;
   gimple pattern_stmt;
   loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
-  struct loop *loop = LOOP_VINFO_LOOP (loop_info);
+  struct loop *loop;
   tree var;
 
+  if (!loop_info)
+    return NULL;
+
+  loop = LOOP_VINFO_LOOP (loop_info);
+
   if (!is_gimple_assign (last_stmt))
     return NULL;
 
@@ -899,7 +920,11 @@ vect_operation_fits_smaller_type (gimple stmt, tree def, tree *new_type,
   gimple def_stmt, new_stmt;
   bool first = false;
   loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (vinfo_for_stmt (stmt));
-  struct loop *loop = LOOP_VINFO_LOOP (loop_info);
+  bb_vec_info bb_info = STMT_VINFO_BB_VINFO (vinfo_for_stmt (stmt));
+  struct loop *loop = NULL;
+
+  if (loop_info)
+    loop = LOOP_VINFO_LOOP (loop_info);
 
   *op0 = NULL_TREE;
   *op1 = NULL_TREE;
@@ -933,7 +958,9 @@ vect_operation_fits_smaller_type (gimple stmt, tree def, tree *new_type,
       first = true;
       if (!widened_name_p (oprnd, stmt, &half_type, &def_stmt, false)
           || !gimple_bb (def_stmt)
-          || !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
+          || (loop && !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
+	  || (!loop && gimple_bb (def_stmt) != BB_VINFO_BB (bb_info)
+	      && gimple_code (def_stmt) != GIMPLE_PHI)
           || !vinfo_for_stmt (def_stmt))
         return false;
     }
@@ -1107,8 +1134,17 @@ vect_recog_over_widening_pattern (VEC (gimple, heap) **stmts,
   int nuses = 0;
   tree var = NULL_TREE, new_type = NULL_TREE, tmp, new_oprnd;
   bool first;
-  struct loop *loop = (gimple_bb (stmt))->loop_father;
   tree type = NULL;
+  loop_vec_info loop_vinfo;
+  struct loop *loop = NULL;
+  bb_vec_info bb_vinfo;
+  stmt_vec_info stmt_vinfo;
+
+  stmt_vinfo = vinfo_for_stmt (stmt);
+  loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+  bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
+  if (loop_vinfo)
+    loop = LOOP_VINFO_LOOP (loop_vinfo);
 
   first = true;
   while (1)
@@ -1141,7 +1177,8 @@ vect_recog_over_widening_pattern (VEC (gimple, heap) **stmts,
 
       if (nuses != 1 || !is_gimple_assign (use_stmt)
           || !gimple_bb (use_stmt)
-          || !flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
+          || (loop && !flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
+	  || (!loop && gimple_bb (use_stmt) != BB_VINFO_BB (bb_vinfo)))
         return NULL;
 
       /* Create pattern statement for STMT.  */
@@ -1518,6 +1555,7 @@ vect_recog_vector_vector_shift_pattern (VEC (gimple, heap) **stmts,
   enum tree_code rhs_code;
   stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
   loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+  bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
   enum vect_def_type dt;
   tree def;
 
@@ -1551,7 +1589,7 @@ vect_recog_vector_vector_shift_pattern (VEC (gimple, heap) **stmts,
 	 != TYPE_PRECISION (TREE_TYPE (oprnd0)))
     return NULL;
 
-  if (!vect_is_simple_use (oprnd1, last_stmt, loop_vinfo, NULL, &def_stmt,
+  if (!vect_is_simple_use (oprnd1, last_stmt, loop_vinfo, bb_vinfo, &def_stmt,
 			   &def, &dt))
     return NULL;
 
@@ -1840,6 +1878,7 @@ vect_recog_mixed_size_cond_pattern (VEC (gimple, heap) **stmts, tree *type_in,
   enum machine_mode cmpmode;
   gimple pattern_stmt, def_stmt;
   loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+  bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
 
   if (!is_gimple_assign (last_stmt)
       || gimple_assign_rhs_code (last_stmt) != COND_EXPR
@@ -1907,12 +1946,15 @@ vect_recog_mixed_size_cond_pattern (VEC (gimple, heap) **stmts, tree *type_in,
 				    gimple_assign_lhs (def_stmt), NULL_TREE);
 
   new_pattern_def_seq (stmt_vinfo, def_stmt);
-  def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
+  def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo, bb_vinfo);
   set_vinfo_for_stmt (def_stmt, def_stmt_info);
   STMT_VINFO_VECTYPE (def_stmt_info) = vecitype;
   *type_in = vecitype;
   *type_out = vectype;
 
+  if (vect_print_dump_info (REPORT_DETAILS))
+    fprintf (vect_dump, "vect_recog_mixed_size_cond_pattern: detected: ");
+
   return pattern_stmt;
 }
 
@@ -1921,14 +1963,15 @@ vect_recog_mixed_size_cond_pattern (VEC (gimple, heap) **stmts, tree *type_in,
    true if bool VAR can be optimized that way.  */
 
 static bool
-check_bool_pattern (tree var, loop_vec_info loop_vinfo)
+check_bool_pattern (tree var, loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
 {
   gimple def_stmt;
   enum vect_def_type dt;
   tree def, rhs1;
   enum tree_code rhs_code;
 
-  if (!vect_is_simple_use (var, NULL, loop_vinfo, NULL, &def_stmt, &def, &dt))
+  if (!vect_is_simple_use (var, NULL, loop_vinfo, bb_vinfo, &def_stmt, &def,
+			   &dt))
     return false;
 
   if (dt != vect_internal_def)
@@ -1945,24 +1988,25 @@ check_bool_pattern (tree var, loop_vec_info loop_vinfo)
   switch (rhs_code)
     {
     case SSA_NAME:
-      return check_bool_pattern (rhs1, loop_vinfo);
+      return check_bool_pattern (rhs1, loop_vinfo, bb_vinfo);
 
     CASE_CONVERT:
       if ((TYPE_PRECISION (TREE_TYPE (rhs1)) != 1
 	   || !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
 	  && TREE_CODE (TREE_TYPE (rhs1)) != BOOLEAN_TYPE)
 	return false;
-      return check_bool_pattern (rhs1, loop_vinfo);
+      return check_bool_pattern (rhs1, loop_vinfo, bb_vinfo);
 
     case BIT_NOT_EXPR:
-      return check_bool_pattern (rhs1, loop_vinfo);
+      return check_bool_pattern (rhs1, loop_vinfo, bb_vinfo);
 
     case BIT_AND_EXPR:
     case BIT_IOR_EXPR:
     case BIT_XOR_EXPR:
-      if (!check_bool_pattern (rhs1, loop_vinfo))
+      if (!check_bool_pattern (rhs1, loop_vinfo, bb_vinfo))
 	return false;
-      return check_bool_pattern (gimple_assign_rhs2 (def_stmt), loop_vinfo);
+      return check_bool_pattern (gimple_assign_rhs2 (def_stmt), loop_vinfo,
+				 bb_vinfo);
 
     default:
       if (TREE_CODE_CLASS (rhs_code) == tcc_comparison)
@@ -2260,6 +2304,7 @@ vect_recog_bool_pattern (VEC (gimple, heap) **stmts, tree *type_in,
   tree var, lhs, rhs, vectype;
   stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
   loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+  bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
   gimple pattern_stmt;
 
   if (!is_gimple_assign (last_stmt))
@@ -2283,7 +2328,7 @@ vect_recog_bool_pattern (VEC (gimple, heap) **stmts, tree *type_in,
       if (vectype == NULL_TREE)
 	return NULL;
 
-      if (!check_bool_pattern (var, loop_vinfo))
+      if (!check_bool_pattern (var, loop_vinfo, bb_vinfo))
 	return NULL;
 
       rhs = adjust_bool_pattern (var, TREE_TYPE (lhs), NULL_TREE, stmts);
@@ -2297,6 +2342,9 @@ vect_recog_bool_pattern (VEC (gimple, heap) **stmts, tree *type_in,
       *type_out = vectype;
       *type_in = vectype;
       VEC_safe_push (gimple, heap, *stmts, last_stmt);
+      if (vect_print_dump_info (REPORT_DETAILS))
+	fprintf (vect_dump, "vect_recog_bool_pattern: detected: ");
+
       return pattern_stmt;
     }
   else if (rhs_code == SSA_NAME
@@ -2307,7 +2355,7 @@ vect_recog_bool_pattern (VEC (gimple, heap) **stmts, tree *type_in,
       gcc_assert (vectype != NULL_TREE);
       if (!VECTOR_MODE_P (TYPE_MODE (vectype)))
 	return NULL;
-      if (!check_bool_pattern (var, loop_vinfo))
+      if (!check_bool_pattern (var, loop_vinfo, bb_vinfo))
 	return NULL;
 
       rhs = adjust_bool_pattern (var, TREE_TYPE (vectype), NULL_TREE, stmts);
@@ -2322,7 +2370,8 @@ vect_recog_bool_pattern (VEC (gimple, heap) **stmts, tree *type_in,
 	}
       pattern_stmt
 	= gimple_build_assign_with_ops (SSA_NAME, lhs, rhs, NULL_TREE);
-      pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo, NULL);
+      pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo,
+						bb_vinfo);
       set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
       STMT_VINFO_DATA_REF (pattern_stmt_info)
 	= STMT_VINFO_DATA_REF (stmt_vinfo);
@@ -2338,6 +2387,8 @@ vect_recog_bool_pattern (VEC (gimple, heap) **stmts, tree *type_in,
       *type_out = vectype;
       *type_in = vectype;
       VEC_safe_push (gimple, heap, *stmts, last_stmt);
+      if (vect_print_dump_info (REPORT_DETAILS))
+	fprintf (vect_dump, "vect_recog_bool_pattern: detected: ");
       return pattern_stmt;
     }
   else
@@ -2354,12 +2405,14 @@ vect_mark_pattern_stmts (gimple orig_stmt, gimple pattern_stmt,
   stmt_vec_info pattern_stmt_info, def_stmt_info;
   stmt_vec_info orig_stmt_info = vinfo_for_stmt (orig_stmt);
   loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (orig_stmt_info);
+  bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (orig_stmt_info);
   gimple def_stmt;
 
   pattern_stmt_info = vinfo_for_stmt (pattern_stmt);
   if (pattern_stmt_info == NULL)
     {
-      pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo, NULL);
+      pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo,
+						bb_vinfo);
       set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
     }
   gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt));
@@ -2382,7 +2435,8 @@ vect_mark_pattern_stmts (gimple orig_stmt, gimple pattern_stmt,
 	  def_stmt_info = vinfo_for_stmt (def_stmt);
 	  if (def_stmt_info == NULL)
 	    {
-	      def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
+	      def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo,
+						 bb_vinfo);
 	      set_vinfo_for_stmt (def_stmt, def_stmt_info);
 	    }
 	  gimple_set_bb (def_stmt, gimple_bb (orig_stmt));
@@ -2493,9 +2547,10 @@ vect_pattern_recog_1 (vect_recog_func_ptr vect_recog_func,
 
   /* Patterns cannot be vectorized using SLP, because they change the order of
      computation.  */
-  FOR_EACH_VEC_ELT (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i, next)
-    if (next == stmt)
-      VEC_ordered_remove (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i); 
+  if (loop_vinfo)
+    FOR_EACH_VEC_ELT (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i, next)
+      if (next == stmt)
+        VEC_ordered_remove (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i);
 
   /* It is possible that additional pattern stmts are created and inserted in
      STMTS_TO_REPLACE.  We create a stmt_info for each of them, and mark the
@@ -2595,19 +2650,34 @@ vect_pattern_recog_1 (vect_recog_func_ptr vect_recog_func,
    be recorded in S3.  */
 
 void
-vect_pattern_recog (loop_vec_info loop_vinfo)
+vect_pattern_recog (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
 {
-  struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-  basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
-  unsigned int nbbs = loop->num_nodes;
+  struct loop *loop;
+  basic_block *bbs, bb;
+  unsigned int nbbs;
   gimple_stmt_iterator si;
   unsigned int i, j;
   vect_recog_func_ptr vect_recog_func;
   VEC (gimple, heap) *stmts_to_replace = VEC_alloc (gimple, heap, 1);
+  gimple stmt;
 
   if (vect_print_dump_info (REPORT_DETAILS))
     fprintf (vect_dump, "=== vect_pattern_recog ===");
 
+  if (loop_vinfo)
+    {
+      loop = LOOP_VINFO_LOOP (loop_vinfo);
+      bbs = LOOP_VINFO_BBS (loop_vinfo);
+      nbbs = loop->num_nodes;
+    }
+  else
+    {
+      bb = BB_VINFO_BB (bb_vinfo);
+      nbbs = 1;
+      bbs = XNEW (basic_block);
+      bbs[0] = bb;
+    }
+
   /* Scan through the loop stmts, applying the pattern recognition
      functions starting at each stmt visited:  */
   for (i = 0; i < nbbs; i++)
@@ -2615,6 +2685,11 @@ vect_pattern_recog (loop_vec_info loop_vinfo)
       basic_block bb = bbs[i];
       for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
         {
+	  if (bb_vinfo && (stmt = gsi_stmt (si))
+	      && vinfo_for_stmt (stmt)
+	      && !STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)))
+	   continue;
+
           /* Scan over all generic vect_recog_xxx_pattern functions.  */
           for (j = 0; j < NUM_PATTERNS; j++)
             {
@@ -2626,4 +2701,6 @@ vect_pattern_recog (loop_vec_info loop_vinfo)
     }
 
   VEC_free (gimple, heap, stmts_to_replace);
+  if (bb_vinfo)
+    free (bbs);
 }
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index d20fc9d..3ba984e 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -249,12 +249,14 @@ vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
       /* Check if DEF_STMT is a part of a pattern in LOOP and get the def stmt
          from the pattern.  Check that all the stmts of the node are in the
          pattern.  */
-      if (loop && def_stmt && gimple_bb (def_stmt)
-          && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
+      if (def_stmt && gimple_bb (def_stmt)
+          && ((loop && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
+	      || (!loop && gimple_bb (def_stmt) == BB_VINFO_BB (bb_vinfo)
+		  && gimple_code (def_stmt) != GIMPLE_PHI))
           && vinfo_for_stmt (def_stmt)
           && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (def_stmt))
-          && !STMT_VINFO_RELEVANT (vinfo_for_stmt (def_stmt))
-          && !STMT_VINFO_LIVE_P (vinfo_for_stmt (def_stmt)))
+	  && !STMT_VINFO_RELEVANT (vinfo_for_stmt (def_stmt))
+	  && !STMT_VINFO_LIVE_P (vinfo_for_stmt (def_stmt)))
         {
           pattern = true;
           if (!first && !oprnd_info->first_pattern)
@@ -2015,7 +2017,9 @@ vect_slp_analyze_bb_1 (basic_block bb)
       return NULL;
     }
 
-   if (!vect_analyze_data_ref_dependences (NULL, bb_vinfo, &max_vf)
+  vect_pattern_recog (NULL, bb_vinfo);
+
+  if (!vect_analyze_data_ref_dependences (NULL, bb_vinfo, &max_vf)
        || min_vf > max_vf)
      {
        if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index 65a70fa..0e7914f 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -2271,10 +2271,10 @@ vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi,
       /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
 	 OP1.  */
       if (CONSTANT_CLASS_P (op0))
-	ok = vect_is_simple_use_1 (op1, stmt, loop_vinfo, NULL,
+	ok = vect_is_simple_use_1 (op1, stmt, loop_vinfo, bb_vinfo,
 				   &def_stmt, &def, &dt[1], &vectype_in);
       else
-	ok = vect_is_simple_use (op1, stmt, loop_vinfo, NULL, &def_stmt,
+	ok = vect_is_simple_use (op1, stmt, loop_vinfo, bb_vinfo, &def_stmt,
 				 &def, &dt[1]);
 
       if (!ok)
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 35420ba..14eb201 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -933,7 +933,7 @@ extern void vect_slp_transform_bb (basic_block);
    in the future.  */
 typedef gimple (* vect_recog_func_ptr) (VEC (gimple, heap) **, tree *, tree *);
 #define NUM_PATTERNS 10
-void vect_pattern_recog (loop_vec_info);
+void vect_pattern_recog (loop_vec_info, bb_vec_info);
 
 /* In tree-vectorizer.c.  */
 unsigned vectorize_loops (void);