diff options
| -rw-r--r-- | gcc/ChangeLog | 13 | ||||
| -rw-r--r-- | gcc/tree-vect-loop.c | 11 | ||||
| -rw-r--r-- | gcc/tree-vect-stmts.c | 41 | ||||
| -rw-r--r-- | gcc/tree-vectorizer.h | 2 |
4 files changed, 57 insertions, 10 deletions
diff --git a/gcc/ChangeLog b/gcc/ChangeLog index cbddb90..506e93d7 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,16 @@ +2019-09-19 Richard Sandiford <richard.sandiford@arm.com> + + * tree-vectorizer.h (vectorizable_condition): Take an int + reduction index instead of a boolean flag. + * tree-vect-stmts.c (vectorizable_condition): Likewise. + Swap the "then" and "else" values for EXTRACT_LAST_REDUCTION + reductions if the reduction accumulator is the "then" rather + than the "else" value. + (vect_analyze_stmt): Update call accordingly. + (vect_transform_stmt): Likewise. + * tree-vect-loop.c (vectorizable_reduction): Likewise, + asserting that the index is > 0. + 2019-09-19 Martin Liska <mliska@suse.cz> * ipa-icf.c (sort_sem_items_by_decl_uid): Simplify comparator. diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index 5ee4ee3..aec4462 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -6659,8 +6659,9 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, { /* Only call during the analysis stage, otherwise we'll lose STMT_VINFO_TYPE. */ + gcc_assert (reduc_index > 0); if (!vec_stmt && !vectorizable_condition (stmt_info, gsi, NULL, - true, NULL, cost_vec)) + reduc_index, NULL, cost_vec)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -7113,9 +7114,9 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, if (reduction_type == EXTRACT_LAST_REDUCTION) { - gcc_assert (!slp_node); + gcc_assert (!slp_node && reduc_index > 0); return vectorizable_condition (stmt_info, gsi, vec_stmt, - true, NULL, NULL); + reduc_index, NULL, NULL); } /* Create the destination vector */ @@ -7145,9 +7146,9 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, { if (code == COND_EXPR) { - gcc_assert (!slp_node); + gcc_assert (!slp_node && reduc_index > 0); vectorizable_condition (stmt_info, gsi, vec_stmt, - true, NULL, NULL); + reduc_index, NULL, NULL); break; } if (code == LSHIFT_EXPR diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 7f5f1c3..0636e5c 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -9778,7 +9778,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, bool vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, bool for_reduction, + stmt_vec_info *vec_stmt, int reduc_index, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { vec_info *vinfo = stmt_info->vinfo; @@ -9807,6 +9807,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, vec<tree> vec_oprnds3 = vNULL; tree vec_cmp_type; bool masked = false; + bool for_reduction = (reduc_index > 0); if (for_reduction && STMT_SLP_TYPE (stmt_info)) return false; @@ -9888,6 +9889,29 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, cond_expr1 = TREE_OPERAND (cond_expr, 1); } + /* For conditional reductions, the "then" value needs to be the candidate + value calculated by this iteration while the "else" value needs to be + the result carried over from previous iterations. If the COND_EXPR + is the other way around, we need to swap it. */ + bool must_invert_cmp_result = false; + if (reduction_type == EXTRACT_LAST_REDUCTION && reduc_index == 1) + { + if (masked) + must_invert_cmp_result = true; + else + { + bool honor_nans = HONOR_NANS (TREE_TYPE (cond_expr0)); + tree_code new_code = invert_tree_comparison (cond_code, honor_nans); + if (new_code == ERROR_MARK) + must_invert_cmp_result = true; + else + cond_code = new_code; + } + /* Make sure we don't accidentally use the old condition. */ + cond_expr = NULL_TREE; + std::swap (then_clause, else_clause); + } + if (!masked && VECTOR_BOOLEAN_TYPE_P (comp_vectype)) { /* Boolean values may have another representation in vectors @@ -10102,6 +10126,15 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, vect_finish_stmt_generation (stmt_info, new_stmt, gsi); vec_compare = vec_compare_name; } + if (must_invert_cmp_result) + { + tree vec_compare_name = make_ssa_name (vec_cmp_type); + gassign *new_stmt = gimple_build_assign (vec_compare_name, + BIT_NOT_EXPR, + vec_compare); + vect_finish_stmt_generation (stmt_info, new_stmt, gsi); + vec_compare = vec_compare_name; + } gcall *new_stmt = gimple_build_call_internal (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare, vec_then_clause); @@ -10635,7 +10668,7 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize, node_instance, cost_vec) || vectorizable_induction (stmt_info, NULL, NULL, node, cost_vec) || vectorizable_shift (stmt_info, NULL, NULL, node, cost_vec) - || vectorizable_condition (stmt_info, NULL, NULL, false, node, + || vectorizable_condition (stmt_info, NULL, NULL, 0, node, cost_vec) || vectorizable_comparison (stmt_info, NULL, NULL, node, cost_vec)); @@ -10654,7 +10687,7 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize, || vectorizable_load (stmt_info, NULL, NULL, node, node_instance, cost_vec) || vectorizable_store (stmt_info, NULL, NULL, node, cost_vec) - || vectorizable_condition (stmt_info, NULL, NULL, false, node, + || vectorizable_condition (stmt_info, NULL, NULL, 0, node, cost_vec) || vectorizable_comparison (stmt_info, NULL, NULL, node, cost_vec)); @@ -10759,7 +10792,7 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, break; case condition_vec_info_type: - done = vectorizable_condition (stmt_info, gsi, &vec_stmt, false, + done = vectorizable_condition (stmt_info, gsi, &vec_stmt, 0, slp_node, NULL); gcc_assert (done); break; diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index ac6e899..aef0ff8 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -1534,7 +1534,7 @@ extern void vect_remove_stores (stmt_vec_info); extern opt_result vect_analyze_stmt (stmt_vec_info, bool *, slp_tree, slp_instance, stmt_vector_for_cost *); extern bool vectorizable_condition (stmt_vec_info, gimple_stmt_iterator *, - stmt_vec_info *, bool, slp_tree, + stmt_vec_info *, int, slp_tree, stmt_vector_for_cost *); extern bool vectorizable_shift (stmt_vec_info, gimple_stmt_iterator *, stmt_vec_info *, slp_tree, |