tree-parloops.c (loop_parallel_p): Call vect_is_simple_reduction with additional parameter.

	* tree-parloops.c (loop_parallel_p): Call vect_is_simple_reduction
	with additional parameter.
	* tree-vectorizer.h (enum vect_def_type): Add new value 
	vect_nested_cycle.
	(enum vect_relevant): Add comments.
	(vect_is_simple_reduction): Add new argument.
	* tree-vect-loop.c (vect_analyze_scalar_cycles_1): Add comments.
	Detect nested cycles.
	(vect_is_simple_reduction): Update documentation, add an argument to
	distinguish inner-loop reduction from nested cycle, detect nested
	cycles, fix printings and indentation, don't swap operands in case
	of nested cycle.
	(get_initial_def_for_reduction): Handle subtraction.
	(vect_create_epilog_for_reduction): Add new argument to specify
	reduction variable.
	(vect_finalize_reduction): Handle subtraction, fix comments.
	(vectorizable_reduction): Handle nested cycles. In case of nested cycle
	keep track of the reduction variable position. Call 
	vect_is_simple_reduction with additional parameter. Use original 
	statement code in reduction epilogue for nested cycle. Call
	vect_create_epilog_for_reduction with additional parameter.
	* tree-vect-patterns.c (vect_recog_dot_prod_pattern): Assert inner-loop
	vectorization.
	(vect_recog_widen_sum_pattern): Likewise.
	* tree-vect-stmts.c (process_use): Distinguish between nested cycles
	and reductions.
	(vect_mark_stmts_to_be_vectorized): Likewise.
	(vect_get_vec_def_for_operand): Handle nested cycles.

From-SVN: r148518

1 files changed, 167 insertions, 57 deletions

diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index 72de3d6..a37e3c0 100644
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -414,7 +414,9 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop)
   if (vect_print_dump_info (REPORT_DETAILS))
     fprintf (vect_dump, "=== vect_analyze_scalar_cycles ===");
 
-  /* First - identify all inductions.  */
+  /* First - identify all inductions. Reduction detection assumes that all the
+     inductions have been identified, therefore, this order must not be 
+     changed.  */
   for (gsi = gsi_start_phis  (bb); !gsi_end_p (gsi); gsi_next (&gsi))
     {
       gimple phi = gsi_stmt (gsi);
@@ -456,13 +458,14 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop)
     }
 
 
-  /* Second - identify all reductions.  */
+  /* Second - identify all reductions and nested cycles.  */
   while (VEC_length (gimple, worklist) > 0)
     {
       gimple phi = VEC_pop (gimple, worklist);
       tree def = PHI_RESULT (phi);
       stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
       gimple reduc_stmt;
+      bool nested_cycle;
 
       if (vect_print_dump_info (REPORT_DETAILS))
         { 
@@ -473,14 +476,28 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop)
       gcc_assert (is_gimple_reg (SSA_NAME_VAR (def)));
       gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
 
-      reduc_stmt = vect_is_simple_reduction (loop_vinfo, phi);
+      nested_cycle = (loop != LOOP_VINFO_LOOP (loop_vinfo));
+      reduc_stmt = vect_is_simple_reduction (loop_vinfo, phi, !nested_cycle);
       if (reduc_stmt)
         {
-          if (vect_print_dump_info (REPORT_DETAILS))
-            fprintf (vect_dump, "Detected reduction.");
-          STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def;
-          STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
+          if (nested_cycle)
+            {
+              if (vect_print_dump_info (REPORT_DETAILS))
+                fprintf (vect_dump, "Detected vectorizable nested cycle.");
+
+              STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_nested_cycle;
+              STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
+                                                        vect_nested_cycle;
+            }
+          else
+            {
+              if (vect_print_dump_info (REPORT_DETAILS))
+                fprintf (vect_dump, "Detected reduction.");
+
+              STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def;
+              STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
                                                         vect_reduction_def;
+            }
         }
       else
         if (vect_print_dump_info (REPORT_DETAILS))
@@ -488,7 +505,6 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop)
     }
 
   VEC_free (gimple, heap, worklist);
-  return;
 }
 
 
@@ -1501,15 +1517,19 @@ report_vect_op (gimple stmt, const char *msg)
   
    such that:
    1. operation is commutative and associative and it is safe to 
-      change the order of the computation.
+      change the order of the computation (if CHECK_REDUCTION is true)
    2. no uses for a2 in the loop (a2 is used out of the loop)
    3. no uses of a1 in the loop besides the reduction operation.
 
    Condition 1 is tested here.
-   Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.  */
+   Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.  
+
+   Also detect a cross-iteration def-use cycle in nested loops, i.e., nested
+   cycles, if CHECK_REDUCTION is false.  */
 
 gimple
-vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
+vect_is_simple_reduction (loop_vec_info loop_info, gimple phi, 
+                          bool check_reduction)
 {
   struct loop *loop = (gimple_bb (phi))->loop_father;
   struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info);
@@ -1524,7 +1544,10 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
   imm_use_iterator imm_iter;
   use_operand_p use_p;
 
-  gcc_assert (loop == vect_loop || flow_loop_nested_p (vect_loop, loop));
+  /* If CHECK_REDUCTION is true, we assume inner-most loop vectorization,
+     otherwise, we assume outer loop vectorization.  */
+  gcc_assert ((check_reduction && loop == vect_loop) 
+              || (!check_reduction && flow_loop_nested_p (vect_loop, loop)));
 
   name = PHI_RESULT (phi);
   nloop_uses = 0;
@@ -1587,7 +1610,8 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
 
   code = gimple_assign_rhs_code (def_stmt);
 
-  if (!commutative_tree_code (code) || !associative_tree_code (code))
+  if (check_reduction 
+      && (!commutative_tree_code (code) || !associative_tree_code (code)))
     {
       if (vect_print_dump_info (REPORT_DETAILS))
         report_vect_op (def_stmt, "reduction: not commutative/associative: ");
@@ -1610,7 +1634,6 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
       return NULL;
     }
 
-  /* Check that it's ok to change the order of the computation.  */
   type = TREE_TYPE (gimple_assign_lhs (def_stmt));
   if (TYPE_MAIN_VARIANT (type) != TYPE_MAIN_VARIANT (TREE_TYPE (op1))
       || TYPE_MAIN_VARIANT (type) != TYPE_MAIN_VARIANT (TREE_TYPE (op2)))
@@ -1627,7 +1650,8 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
       return NULL;
     }
 
-  /* Generally, when vectorizing a reduction we change the order of the
+  /* Check that it's ok to change the order of the computation.  
+     Generally, when vectorizing a reduction we change the order of the
      computation.  This may change the behavior of the program in some
      cases, so we need to check that this is ok.  One exception is when 
      vectorizing an outer-loop: the inner-loop is executed sequentially,
@@ -1636,7 +1660,7 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
 
   /* CHECKME: check for !flag_finite_math_only too?  */
   if (SCALAR_FLOAT_TYPE_P (type) && !flag_associative_math
-      && !nested_in_vect_loop_p (vect_loop, def_stmt)) 
+      && check_reduction) 
     {
       /* Changing the order of operations changes the semantics.  */
       if (vect_print_dump_info (REPORT_DETAILS))
@@ -1644,14 +1668,14 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
       return NULL;
     }
   else if (INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type)
-	   && !nested_in_vect_loop_p (vect_loop, def_stmt))
+	   && check_reduction)
     {
       /* Changing the order of operations changes the semantics.  */
       if (vect_print_dump_info (REPORT_DETAILS))
 	report_vect_op (def_stmt, "reduction: unsafe int math optimization: ");
       return NULL;
     }
-  else if (SAT_FIXED_POINT_TYPE_P (type))
+  else if (SAT_FIXED_POINT_TYPE_P (type) && check_reduction)
     {
       /* Changing the order of operations changes the semantics.  */
       if (vect_print_dump_info (REPORT_DETAILS))
@@ -1660,10 +1684,10 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
       return NULL;
     }
 
-  /* reduction is safe. we're dealing with one of the following:
+  /* Reduction is safe. We're dealing with one of the following:
      1) integer arithmetic and no trapv
-     2) floating point arithmetic, and special flags permit this optimization.
-   */
+     2) floating point arithmetic, and special flags permit this optimization
+     3) nested cycle (i.e., outer loop vectorization).  */
   def1 = SSA_NAME_DEF_STMT (op1);
   def2 = SSA_NAME_DEF_STMT (op2);
   if (!def1 || !def2 || gimple_nop_p (def1) || gimple_nop_p (def2))
@@ -1683,35 +1707,49 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
       && (is_gimple_assign (def1)
 	  || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1)) == vect_induction_def
 	  || (gimple_code (def1) == GIMPLE_PHI
-	      && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1)) == vect_internal_def
+	      && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1)) 
+                  == vect_internal_def
 	      && !is_loop_header_bb_p (gimple_bb (def1)))))
     {
       if (vect_print_dump_info (REPORT_DETAILS))
-	report_vect_op (def_stmt, "detected reduction:");
+	report_vect_op (def_stmt, "detected reduction: ");
       return def_stmt;
     }
   else if (def1 == phi
 	   && flow_bb_inside_loop_p (loop, gimple_bb (def2))
 	   && (is_gimple_assign (def2)
-	       || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2)) == vect_induction_def
+	       || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2))
+                    == vect_induction_def
 	       || (gimple_code (def2) == GIMPLE_PHI
-		   && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2)) == vect_internal_def
+		   && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2)) 
+                    == vect_internal_def
 		   && !is_loop_header_bb_p (gimple_bb (def2)))))
     {
-      /* Swap operands (just for simplicity - so that the rest of the code
-	 can assume that the reduction variable is always the last (second)
-	 argument).  */
-      if (vect_print_dump_info (REPORT_DETAILS))
-	report_vect_op (def_stmt ,
-		        "detected reduction: need to swap operands:");
-      swap_tree_operands (def_stmt, gimple_assign_rhs1_ptr (def_stmt),
-			  gimple_assign_rhs2_ptr (def_stmt));
+      if (check_reduction)
+        {
+          /* Swap operands (just for simplicity - so that the rest of the code
+	     can assume that the reduction variable is always the last (second)
+	     argument).  */
+          if (vect_print_dump_info (REPORT_DETAILS))
+	    report_vect_op (def_stmt,
+	  	            "detected reduction: need to swap operands: ");
+
+          swap_tree_operands (def_stmt, gimple_assign_rhs1_ptr (def_stmt),
+ 			      gimple_assign_rhs2_ptr (def_stmt));
+        }
+      else
+        {
+          if (vect_print_dump_info (REPORT_DETAILS))
+            report_vect_op (def_stmt, "detected reduction: ");
+        }
+
       return def_stmt;
     }
   else
     {
       if (vect_print_dump_info (REPORT_DETAILS))
-	report_vect_op (def_stmt, "reduction: unknown pattern.");
+	report_vect_op (def_stmt, "reduction: unknown pattern: ");
+
       return NULL;
     }
 }
@@ -2529,6 +2567,7 @@ get_initial_def_for_reduction (gimple stmt, tree init_val, tree *adjustment_def)
   case WIDEN_SUM_EXPR:
   case DOT_PROD_EXPR:
   case PLUS_EXPR:
+  case MINUS_EXPR:
     if (nested_in_vect_loop)
       *adjustment_def = vect_get_vec_def_for_operand (init_val, stmt, NULL);
     else
@@ -2572,6 +2611,8 @@ get_initial_def_for_reduction (gimple stmt, tree init_val, tree *adjustment_def)
      in vectorizable_operation.
    STMT is the scalar reduction stmt that is being vectorized.
    REDUCTION_PHI is the phi-node that carries the reduction computation.
+   REDUC_INDEX is the index of the operand in the right hand side of the 
+     statement that is defined by REDUCTION_PHI.
 
    This function:
    1. Creates the reduction def-use cycle: sets the arguments for 
@@ -2615,7 +2656,8 @@ static void
 vect_create_epilog_for_reduction (tree vect_def, gimple stmt,
 				  int ncopies,
 				  enum tree_code reduc_code,
-				  gimple reduction_phi)
+				  gimple reduction_phi,
+                                  int reduc_index)
 {
   stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
   stmt_vec_info prev_phi_info;
@@ -2659,14 +2701,16 @@ vect_create_epilog_for_reduction (tree vect_def, gimple stmt,
   switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt)))
     {
     case GIMPLE_SINGLE_RHS:
-      gcc_assert (TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt)) == ternary_op);
-      reduction_op = TREE_OPERAND (gimple_assign_rhs1 (stmt), 2);
+      gcc_assert (TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt)) 
+                                       == ternary_op);
+      reduction_op = TREE_OPERAND (gimple_assign_rhs1 (stmt), reduc_index);
       break;
     case GIMPLE_UNARY_RHS:
       reduction_op = gimple_assign_rhs1 (stmt);
       break;
     case GIMPLE_BINARY_RHS:
-      reduction_op = gimple_assign_rhs2 (stmt);
+      reduction_op = reduc_index ? 
+                     gimple_assign_rhs2 (stmt) : gimple_assign_rhs1 (stmt);
       break;
     default:
       gcc_unreachable ();
@@ -2754,6 +2798,7 @@ vect_create_epilog_for_reduction (tree vect_def, gimple stmt,
       SET_PHI_ARG_DEF (phi, single_exit (loop)->dest_idx, def);
       prev_phi_info = vinfo_for_stmt (phi);
     }
+
   exit_gsi = gsi_after_labels (exit_bb);
 
   /* 2.2 Get the relevant tree-code to use in the epilog for schemes 2,3 
@@ -2778,6 +2823,7 @@ vect_create_epilog_for_reduction (tree vect_def, gimple stmt,
       gcc_assert (STMT_VINFO_IN_PATTERN_P (stmt_vinfo));
       gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo) == stmt);
     }
+
   code = gimple_assign_rhs_code (orig_stmt);
   scalar_dest = gimple_assign_lhs (orig_stmt);
   scalar_type = TREE_TYPE (scalar_dest);
@@ -2970,6 +3016,11 @@ vect_finalize_reduction:
     {
       if (nested_in_vect_loop)
 	{
+          /* For MINUS_EXPR we create new_temp = loop_exit_def + adjustment_def
+             since the initial value is [0,0,...,0].  */
+          if (code == MINUS_EXPR)
+            code = PLUS_EXPR;
+
 	  gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def)) == VECTOR_TYPE);
 	  expr = build2 (code, vectype, PHI_RESULT (new_phi), adjustment_def);
 	  new_dest = vect_create_destination_var (scalar_dest, vectype);
@@ -2980,6 +3031,7 @@ vect_finalize_reduction:
 	  expr = build2 (code, scalar_type, new_temp, adjustment_def);
 	  new_dest = vect_create_destination_var (scalar_dest, scalar_type);
 	}
+
       epilog_stmt = gimple_build_assign (new_dest, expr);
       new_temp = make_ssa_name (new_dest, epilog_stmt);
       gimple_assign_set_lhs (epilog_stmt, new_temp);
@@ -3043,7 +3095,7 @@ vect_finalize_reduction:
 
    Check if STMT performs a reduction operation that can be vectorized.
    If VEC_STMT is also passed, vectorize the STMT: create a vectorized
-   stmt to replace it, put it in VEC_STMT, and insert it at BSI.
+   stmt to replace it, put it in VEC_STMT, and insert it at GSI.
    Return FALSE if not a vectorizable STMT, TRUE otherwise.
 
    This function also handles reduction idioms (patterns) that have been 
@@ -3115,9 +3167,16 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
   gimple new_stmt = NULL;
   int j;
   tree ops[3];
+  bool nested_cycle = false, found_nested_cycle_def = false;
+  gimple reduc_def_stmt = NULL;
+  /* The default is that the reduction variable is the last in statement.  */
+  int reduc_index = 2;
 
   if (nested_in_vect_loop_p (loop, stmt))
-    loop = loop->inner;
+    {
+      loop = loop->inner;
+      nested_cycle = true;
+    }
 
   gcc_assert (ncopies >= 1);
 
@@ -3138,7 +3197,8 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
     return false;
 
   /* Make sure it was already recognized as a reduction computation.  */
-  if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def)
+  if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def
+      && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle)
     return false;
 
   /* 2. Has this been recognized as a reduction pattern? 
@@ -3202,7 +3262,9 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
     return false;
 
   /* All uses but the last are expected to be defined in the loop.
-     The last use is the reduction variable.  */
+     The last use is the reduction variable. In case of nested cycle this 
+     assumption is not true: we use reduc_index to record the index of the
+     reduction variable.  */
   for (i = 0; i < op_type-1; i++)
     {
       is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, NULL, &def_stmt,
@@ -3211,21 +3273,39 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
       if (dt != vect_internal_def
 	  && dt != vect_external_def
 	  && dt != vect_constant_def
-	  && dt != vect_induction_def)
+	  && dt != vect_induction_def
+          && dt != vect_nested_cycle)
 	return false;
+
+      if (dt == vect_nested_cycle)
+        {
+          found_nested_cycle_def = true;
+          reduc_def_stmt = def_stmt;
+          reduc_index = i;
+        }
     }
 
   is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, NULL, &def_stmt, 
                                       &def, &dt);
   gcc_assert (is_simple_use);
-  gcc_assert (dt == vect_reduction_def);
-  gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
+  gcc_assert (dt == vect_reduction_def
+              || dt == vect_nested_cycle
+              || ((dt == vect_internal_def || dt == vect_external_def 
+                   || dt == vect_constant_def || dt == vect_induction_def)
+                   && nested_cycle && found_nested_cycle_def)); 
+  if (!found_nested_cycle_def)
+    reduc_def_stmt = def_stmt;
+
+  gcc_assert (gimple_code (reduc_def_stmt) == GIMPLE_PHI);
   if (orig_stmt) 
-    gcc_assert (orig_stmt == vect_is_simple_reduction (loop_vinfo, def_stmt));
+    gcc_assert (orig_stmt == vect_is_simple_reduction (loop_vinfo, 
+                                                       reduc_def_stmt, 
+                                                       !nested_cycle));
   else
-    gcc_assert (stmt == vect_is_simple_reduction (loop_vinfo, def_stmt));
+    gcc_assert (stmt == vect_is_simple_reduction (loop_vinfo, reduc_def_stmt, 
+                                                  !nested_cycle));
   
-  if (STMT_VINFO_LIVE_P (vinfo_for_stmt (def_stmt)))
+  if (STMT_VINFO_LIVE_P (vinfo_for_stmt (reduc_def_stmt)))
     return false;
 
   /* 4. Supportable by target?  */
@@ -3320,8 +3400,12 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
       orig_code = code;
     }
 
-  if (!reduction_code_for_scalar_code (orig_code, &epilog_reduc_code))
-    return false;
+  if (nested_cycle)
+    epilog_reduc_code = orig_code;
+  else
+    if (!reduction_code_for_scalar_code (orig_code, &epilog_reduc_code))
+      return false;
+
   reduc_optab = optab_for_tree_code (epilog_reduc_code, vectype, optab_default);
   if (!reduc_optab)
     {
@@ -3402,13 +3486,20 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
       /* Handle uses.  */
       if (j == 0)
         {
-	  loop_vec_def0 = vect_get_vec_def_for_operand (ops[0], stmt, NULL);
+	  loop_vec_def0 = vect_get_vec_def_for_operand (ops[!reduc_index], 
+                                                        stmt, NULL);
           if (op_type == ternary_op)
             {
-	      loop_vec_def1 = vect_get_vec_def_for_operand (ops[1], stmt, NULL);
+              if (reduc_index == 0)
+ 	        loop_vec_def1 = vect_get_vec_def_for_operand (ops[2], stmt, 
+                                                              NULL);
+              else
+                loop_vec_def1 = vect_get_vec_def_for_operand (ops[1], stmt, 
+                                                              NULL);
             }
 
-          /* Get the vector def for the reduction variable from the phi node */
+          /* Get the vector def for the reduction variable from the phi 
+             node.  */
           reduc_def = PHI_RESULT (new_phi);
 	  first_phi = new_phi;
         }
@@ -3427,12 +3518,31 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
 	  STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi;
         }
 
+      
       /* Arguments are ready. create the new vector stmt.  */
       if (op_type == binary_op)
-        expr = build2 (code, vectype, loop_vec_def0, reduc_def);
+        {
+          if (reduc_index == 0)
+            expr = build2 (code, vectype, reduc_def, loop_vec_def0);
+          else
+            expr = build2 (code, vectype, loop_vec_def0, reduc_def);
+        }
       else
-        expr = build3 (code, vectype, loop_vec_def0, loop_vec_def1, 
-		       reduc_def);
+        {
+          if (reduc_index == 0)
+            expr = build3 (code, vectype, reduc_def, loop_vec_def0, 
+                           loop_vec_def1);
+          else 
+            {
+              if (reduc_index == 1)
+                expr = build3 (code, vectype, loop_vec_def0, reduc_def, 
+                               loop_vec_def1);
+              else
+                expr = build3 (code, vectype, loop_vec_def0, loop_vec_def1, 
+	     	               reduc_def);
+            }
+        }
+
       new_stmt = gimple_build_assign (vec_dest, expr);
       new_temp = make_ssa_name (vec_dest, new_stmt);
       gimple_assign_set_lhs (new_stmt, new_temp);
@@ -3451,7 +3561,7 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
   if (!single_defuse_cycle)
     new_temp = gimple_assign_lhs (*vec_stmt);
   vect_create_epilog_for_reduction (new_temp, stmt, epilog_copies,
-				    epilog_reduc_code, first_phi);
+				    epilog_reduc_code, first_phi, reduc_index);
   return true;
 }