diff options
Diffstat (limited to 'gcc/tree-vect-slp.cc')
| -rw-r--r-- | gcc/tree-vect-slp.cc | 103 |
1 files changed, 73 insertions, 30 deletions
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc index fb2262a..7a828ca 100644 --- a/gcc/tree-vect-slp.cc +++ b/gcc/tree-vect-slp.cc @@ -507,11 +507,6 @@ vect_def_types_match (enum vect_def_type dta, enum vect_def_type dtb) && (dtb == vect_external_def || dtb == vect_constant_def))); } -static const int cond_expr_maps[3][5] = { - { 4, -1, -2, 1, 2 }, - { 4, -2, -1, 1, 2 }, - { 4, -1, -2, 2, 1 } -}; static const int no_arg_map[] = { 0 }; static const int arg0_map[] = { 1, 0 }; static const int arg1_map[] = { 1, 1 }; @@ -986,13 +981,18 @@ vect_get_and_check_slp_defs (vec_info *vinfo, unsigned char swap, to be combined into the same SLP group. */ bool -compatible_calls_p (gcall *call1, gcall *call2) +compatible_calls_p (gcall *call1, gcall *call2, bool allow_two_operators) { unsigned int nargs = gimple_call_num_args (call1); if (nargs != gimple_call_num_args (call2)) return false; - if (gimple_call_combined_fn (call1) != gimple_call_combined_fn (call2)) + auto cfn1 = gimple_call_combined_fn (call1); + auto cfn2 = gimple_call_combined_fn (call2); + if (cfn1 != cfn2 + && (!allow_two_operators + || !((cfn1 == CFN_FMA || cfn1 == CFN_FMS) + && (cfn2 == CFN_FMA || cfn2 == CFN_FMS)))) return false; if (gimple_call_internal_p (call1)) @@ -1354,10 +1354,14 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap, || rhs_code != IMAGPART_EXPR) /* Handle mismatches in plus/minus by computing both and merging the results. */ - && !((first_stmt_code == PLUS_EXPR - || first_stmt_code == MINUS_EXPR) - && (alt_stmt_code == PLUS_EXPR - || alt_stmt_code == MINUS_EXPR) + && !((((first_stmt_code == PLUS_EXPR + || first_stmt_code == MINUS_EXPR) + && (alt_stmt_code == PLUS_EXPR + || alt_stmt_code == MINUS_EXPR)) + || ((first_stmt_code == CFN_FMA + || first_stmt_code == CFN_FMS) + && (alt_stmt_code == CFN_FMA + || alt_stmt_code == CFN_FMS))) && rhs_code == alt_stmt_code) && !(first_stmt_code.is_tree_code () && rhs_code.is_tree_code () @@ -1406,7 +1410,7 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap, { if (!is_a <gcall *> (stmts[0]->stmt) || !compatible_calls_p (as_a <gcall *> (stmts[0]->stmt), - call_stmt)) + call_stmt, true)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -2845,9 +2849,10 @@ out: && matches[0] /* ??? For COND_EXPRs we can swap the comparison operands as well as the arms under some constraints. */ - && nops == 2 + && (nops == 2 || nops == 3) && oprnds_info[1]->first_dt == vect_internal_def - && is_gimple_assign (stmt_info->stmt) + && (is_gimple_assign (stmt_info->stmt) + || is_gimple_call (stmt_info->stmt)) /* Swapping operands for reductions breaks assumptions later on. */ && STMT_VINFO_REDUC_IDX (stmt_info) == -1) { @@ -2862,14 +2867,32 @@ out: continue; stmt_vec_info stmt_info = stmts[j]; /* Verify if we can swap operands of this stmt. */ - gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt); - if (!stmt - || !commutative_tree_code (gimple_assign_rhs_code (stmt))) + if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt)) { - if (!swap_not_matching) - goto fail; - swap_not_matching = false; - break; + tree_code code = gimple_assign_rhs_code (stmt); + if (! commutative_tree_code (code) + && ! commutative_ternary_tree_code (code)) + { + if (!swap_not_matching) + goto fail; + swap_not_matching = false; + break; + } + } + else if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt)) + { + internal_fn fn = (gimple_call_internal_p (call) + ? gimple_call_internal_fn (call) + : IFN_LAST); + if ((! commutative_binary_fn_p (fn) + && ! commutative_ternary_fn_p (fn)) + || first_commutative_argument (fn) != 0) + { + if (!swap_not_matching) + goto fail; + swap_not_matching = false; + break; + } } } } @@ -3055,24 +3078,35 @@ fail: SLP_TREE_CODE (node) = VEC_PERM_EXPR; SLP_TREE_CHILDREN (node).quick_push (one); SLP_TREE_CHILDREN (node).quick_push (two); - gassign *stmt = as_a <gassign *> (stmts[0]->stmt); - enum tree_code code0 = gimple_assign_rhs_code (stmt); + enum tree_code code0 = ERROR_MARK; enum tree_code ocode = ERROR_MARK; + if (gassign *stmt = dyn_cast <gassign *> (stmts[0]->stmt)) + code0 = gimple_assign_rhs_code (stmt); stmt_vec_info ostmt_info; unsigned j = 0; FOR_EACH_VEC_ELT (stmts, i, ostmt_info) { - gassign *ostmt = as_a <gassign *> (ostmt_info->stmt); - if (gimple_assign_rhs_code (ostmt) != code0) + int op = 0; + if (gassign *ostmt = dyn_cast <gassign *> (ostmt_info->stmt)) { - SLP_TREE_LANE_PERMUTATION (node).safe_push (std::make_pair (1, i)); - ocode = gimple_assign_rhs_code (ostmt); - j = i; + if (gimple_assign_rhs_code (ostmt) != code0) + { + ocode = gimple_assign_rhs_code (ostmt); + op = 1; + j = i; + } } else - SLP_TREE_LANE_PERMUTATION (node).safe_push (std::make_pair (0, i)); + { + if (gimple_call_combined_fn (stmts[0]->stmt) + != gimple_call_combined_fn (ostmt_info->stmt)) + { + op = 1; + j = i; + } + } + SLP_TREE_LANE_PERMUTATION (node).safe_push (std::make_pair (op, i)); } - SLP_TREE_CODE (one) = code0; SLP_TREE_CODE (two) = ocode; SLP_TREE_LANES (one) = stmts.length (); @@ -4557,6 +4591,15 @@ vect_lower_load_permutations (loop_vec_info loop_vinfo, if (!SLP_TREE_CHILDREN (load).is_empty ()) continue; + /* For single-element interleaving spanning multiple vectors avoid + lowering, we want to use VMAT_ELEMENTWISE later. */ + if (ld_lanes_lanes == 0 + && SLP_TREE_LANES (load) == 1 + && !DR_GROUP_NEXT_ELEMENT (first) + && maybe_gt (group_lanes, + TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (load)))) + return; + /* We want to pattern-match special cases here and keep those alone. Candidates are splats and load-lane. */ |