diff options
Diffstat (limited to 'gcc/tree-vect-patterns.c')
| -rw-r--r-- | gcc/tree-vect-patterns.c | 337 |
1 files changed, 132 insertions, 205 deletions
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index 3d7f866..a07b1a9 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -47,66 +47,6 @@ along with GCC; see the file COPYING3. If not see #include "omp-simd-clone.h" #include "predict.h" -/* Pattern recognition functions */ -static gimple *vect_recog_widen_sum_pattern (vec<gimple *> *, tree *, - tree *); -static gimple *vect_recog_widen_mult_pattern (vec<gimple *> *, tree *, - tree *); -static gimple *vect_recog_dot_prod_pattern (vec<gimple *> *, tree *, - tree *); -static gimple *vect_recog_sad_pattern (vec<gimple *> *, tree *, - tree *); -static gimple *vect_recog_pow_pattern (vec<gimple *> *, tree *, tree *); -static gimple *vect_recog_over_widening_pattern (vec<gimple *> *, tree *, - tree *); -static gimple *vect_recog_widen_shift_pattern (vec<gimple *> *, - tree *, tree *); -static gimple *vect_recog_rotate_pattern (vec<gimple *> *, tree *, tree *); -static gimple *vect_recog_vector_vector_shift_pattern (vec<gimple *> *, - tree *, tree *); -static gimple *vect_recog_divmod_pattern (vec<gimple *> *, - tree *, tree *); - -static gimple *vect_recog_mult_pattern (vec<gimple *> *, - tree *, tree *); - -static gimple *vect_recog_mixed_size_cond_pattern (vec<gimple *> *, - tree *, tree *); -static gimple *vect_recog_bool_pattern (vec<gimple *> *, tree *, tree *); -static gimple *vect_recog_mask_conversion_pattern (vec<gimple *> *, tree *, tree *); -static gimple *vect_recog_gather_scatter_pattern (vec<gimple *> *, tree *, - tree *); - -struct vect_recog_func -{ - vect_recog_func_ptr fn; - const char *name; -}; - -/* Note that ordering matters - the first pattern matching on a stmt - is taken which means usually the more complex one needs to preceed - the less comples onex (widen_sum only after dot_prod or sad for example). */ -static vect_recog_func vect_vect_recog_func_ptrs[NUM_PATTERNS] = { - { vect_recog_widen_mult_pattern, "widen_mult" }, - { vect_recog_dot_prod_pattern, "dot_prod" }, - { vect_recog_sad_pattern, "sad" }, - { vect_recog_widen_sum_pattern, "widen_sum" }, - { vect_recog_pow_pattern, "pow" }, - { vect_recog_widen_shift_pattern, "widen_shift" }, - { vect_recog_over_widening_pattern, "over_widening" }, - { vect_recog_rotate_pattern, "rotate" }, - { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" }, - { vect_recog_divmod_pattern, "divmod" }, - { vect_recog_mult_pattern, "mult" }, - { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" }, - { vect_recog_bool_pattern, "bool" }, - /* This must come before mask conversion, and includes the parts - of mask conversion that are needed for gather and scatter - internal functions. */ - { vect_recog_gather_scatter_pattern, "gather_scatter" }, - { vect_recog_mask_conversion_pattern, "mask_conversion" } -}; - /* Report that we've found an instance of pattern PATTERN in statement STMT. */ @@ -134,6 +74,49 @@ new_pattern_def_seq (stmt_vec_info stmt_info, gimple *stmt) append_pattern_def_seq (stmt_info, stmt); } +/* Return true if the target supports a vector version of CODE, + where CODE is known to map to a direct optab. ITYPE specifies + the type of (some of) the scalar inputs and OTYPE specifies the + type of the scalar result. + + If CODE allows the inputs and outputs to have different type + (such as for WIDEN_SUM_EXPR), it is the input mode rather + than the output mode that determines the appropriate target pattern. + Operand 0 of the target pattern then specifies the mode that the output + must have. + + When returning true, set *VECOTYPE_OUT to the vector version of OTYPE. + Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT + is nonnull. */ + +static bool +vect_supportable_direct_optab_p (tree otype, tree_code code, + tree itype, tree *vecotype_out, + tree *vecitype_out = NULL) +{ + tree vecitype = get_vectype_for_scalar_type (itype); + if (!vecitype) + return false; + + tree vecotype = get_vectype_for_scalar_type (otype); + if (!vecotype) + return false; + + optab optab = optab_for_tree_code (code, vecitype, optab_default); + if (!optab) + return false; + + insn_code icode = optab_handler (optab, TYPE_MODE (vecitype)); + if (icode == CODE_FOR_nothing + || insn_data[icode].operand[0].mode != TYPE_MODE (vecotype)) + return false; + + *vecotype_out = vecotype; + if (vecitype_out) + *vecitype_out = vecitype; + return true; +} + /* Check whether STMT2 is in the same loop or basic block as STMT1. Which of the two applies depends on whether we're currently doing loop-based or basic-block-based vectorization, as determined by @@ -293,8 +276,6 @@ vect_reassociating_reduction_p (stmt_vec_info stmt_vinfo) Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the sequence of @@ -310,8 +291,7 @@ vect_reassociating_reduction_p (stmt_vec_info stmt_vinfo) inner-loop nested in an outer-loop that us being vectorized). */ static gimple * -vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *stmt, *last_stmt = (*stmts)[0]; tree oprnd0, oprnd1; @@ -448,8 +428,9 @@ vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in, vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt); half_type = TREE_TYPE (oprnd00); - *type_in = half_type; - *type_out = type; + if (!vect_supportable_direct_optab_p (type, DOT_PROD_EXPR, half_type, + type_out)) + return NULL; var = vect_recog_temp_ssa_var (type, NULL); pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR, @@ -489,8 +470,6 @@ vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in, Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the sequence of @@ -499,8 +478,7 @@ vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in, */ static gimple * -vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = (*stmts)[0]; tree sad_oprnd0, sad_oprnd1; @@ -641,8 +619,9 @@ vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_in, vect_pattern_detected ("vect_recog_sad_pattern", last_stmt); - *type_in = TREE_TYPE (sad_oprnd0); - *type_out = sum_type; + if (!vect_supportable_direct_optab_p (sum_type, SAD_EXPR, + TREE_TYPE (sad_oprnd0), type_out)) + return NULL; tree var = vect_recog_temp_ssa_var (sum_type, NULL); gimple *pattern_stmt = gimple_build_assign (var, SAD_EXPR, sad_oprnd0, @@ -778,8 +757,6 @@ vect_handle_widen_op_by_const (gimple *stmt, enum tree_code code, Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the sequence of @@ -790,8 +767,7 @@ vect_handle_widen_op_by_const (gimple *stmt, enum tree_code code, */ static gimple * -vect_recog_widen_mult_pattern (vec<gimple *> *stmts, - tree *type_in, tree *type_out) +vect_recog_widen_mult_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); gimple *def_stmt0, *def_stmt1; @@ -933,8 +909,9 @@ vect_recog_widen_mult_pattern (vec<gimple *> *stmts, &dummy_int, &dummy_vec)) return NULL; - *type_in = vectype; *type_out = get_vectype_for_scalar_type (type); + if (!*type_out) + return NULL; /* Pattern supported. Create a stmt to be used to replace the pattern: */ var = vect_recog_temp_ssa_var (itype, NULL); @@ -989,8 +966,6 @@ vect_recog_widen_mult_pattern (vec<gimple *> *stmts, Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the sequence of @@ -1001,8 +976,7 @@ vect_recog_widen_mult_pattern (vec<gimple *> *stmts, */ static gimple * -vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = (*stmts)[0]; tree base, exp; @@ -1072,12 +1046,13 @@ vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in, if (node->simd_clones == NULL) return NULL; } + *type_out = get_vectype_for_scalar_type (TREE_TYPE (base)); + if (!*type_out) + return NULL; stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt); tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL); gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc); new_pattern_def_seq (stmt_vinfo, g); - *type_in = TREE_TYPE (base); - *type_out = NULL_TREE; tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL); g = gimple_build_call (exp_decl, 1, def); gimple_call_set_lhs (g, res); @@ -1091,15 +1066,15 @@ vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in, /* We now have a pow or powi builtin function call with a constant exponent. */ - *type_out = NULL_TREE; - /* Catch squaring. */ if ((tree_fits_shwi_p (exp) && tree_to_shwi (exp) == 2) || (TREE_CODE (exp) == REAL_CST && real_equal (&TREE_REAL_CST (exp), &dconst2))) { - *type_in = TREE_TYPE (base); + if (!vect_supportable_direct_optab_p (TREE_TYPE (base), MULT_EXPR, + TREE_TYPE (base), type_out)) + return NULL; var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL); stmt = gimple_build_assign (var, MULT_EXPR, base, base); @@ -1110,9 +1085,9 @@ vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in, if (TREE_CODE (exp) == REAL_CST && real_equal (&TREE_REAL_CST (exp), &dconsthalf)) { - *type_in = get_vectype_for_scalar_type (TREE_TYPE (base)); - if (*type_in - && direct_internal_fn_supported_p (IFN_SQRT, *type_in, + *type_out = get_vectype_for_scalar_type (TREE_TYPE (base)); + if (*type_out + && direct_internal_fn_supported_p (IFN_SQRT, *type_out, OPTIMIZE_FOR_SPEED)) { gcall *stmt = gimple_build_call_internal (IFN_SQRT, 1, base); @@ -1150,8 +1125,6 @@ vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in, Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the sequence of @@ -1167,8 +1140,7 @@ vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in, inner-loop nested in an outer-loop that us being vectorized). */ static gimple * -vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *stmt, *last_stmt = (*stmts)[0]; tree oprnd0, oprnd1; @@ -1223,14 +1195,15 @@ vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_in, if (!type_conversion_p (oprnd0, last_stmt, true, &half_type, &stmt, &promotion) || !promotion) - return NULL; + return NULL; oprnd0 = gimple_assign_rhs1 (stmt); vect_pattern_detected ("vect_recog_widen_sum_pattern", last_stmt); - *type_in = half_type; - *type_out = type; + if (!vect_supportable_direct_optab_p (type, WIDEN_SUM_EXPR, half_type, + type_out)) + return NULL; var = vect_recog_temp_ssa_var (type, NULL); pattern_stmt = gimple_build_assign (var, WIDEN_SUM_EXPR, oprnd0, oprnd1); @@ -1473,8 +1446,7 @@ vect_operation_fits_smaller_type (gimple *stmt, tree def, tree *new_type, demotion operation. We also check that S3 and S4 have only one use. */ static gimple * -vect_recog_over_widening_pattern (vec<gimple *> *stmts, - tree *type_in, tree *type_out) +vect_recog_over_widening_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *stmt = stmts->pop (); gimple *pattern_stmt = NULL, *new_def_stmt, *prev_stmt = NULL, @@ -1562,14 +1534,15 @@ vect_recog_over_widening_pattern (vec<gimple *> *stmts, if (TYPE_UNSIGNED (new_type) != TYPE_UNSIGNED (use_type) || TYPE_PRECISION (new_type) != TYPE_PRECISION (use_type)) { + *type_out = get_vectype_for_scalar_type (use_type); + if (!*type_out) + return NULL; + /* Create NEW_TYPE->USE_TYPE conversion. */ new_oprnd = make_ssa_name (use_type); pattern_stmt = gimple_build_assign (new_oprnd, NOP_EXPR, var); STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use_stmt)) = pattern_stmt; - *type_in = get_vectype_for_scalar_type (new_type); - *type_out = get_vectype_for_scalar_type (use_type); - /* We created a pattern statement for the last statement in the sequence, so we don't need to associate it with the pattern statement created for PREV_STMT. Therefore, we add PREV_STMT @@ -1583,8 +1556,7 @@ vect_recog_over_widening_pattern (vec<gimple *> *stmts, STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (use_stmt)) = STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (prev_stmt)); - *type_in = vectype; - *type_out = NULL_TREE; + *type_out = vectype; } stmts->safe_push (use_stmt); @@ -1648,8 +1620,6 @@ vect_recog_over_widening_pattern (vec<gimple *> *stmts, Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the sequence of @@ -1657,8 +1627,7 @@ vect_recog_over_widening_pattern (vec<gimple *> *stmts, WIDEN_LSHIFT_EXPR <a_t, CONST>. */ static gimple * -vect_recog_widen_shift_pattern (vec<gimple *> *stmts, - tree *type_in, tree *type_out) +vect_recog_widen_shift_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); gimple *def_stmt0; @@ -1740,7 +1709,6 @@ vect_recog_widen_shift_pattern (vec<gimple *> *stmts, &dummy_int, &dummy_vec)) return NULL; - *type_in = vectype; *type_out = vectype_out; /* Pattern supported. Create a stmt to be used to replace the pattern. */ @@ -1783,15 +1751,13 @@ vect_recog_widen_shift_pattern (vec<gimple *> *stmts, Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the rotate S0 stmt. */ static gimple * -vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_in, tree *type_out) +vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2; @@ -1876,10 +1842,7 @@ vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_in, tree *type_out) return NULL; } - *type_in = vectype; *type_out = vectype; - if (*type_in == NULL_TREE) - return NULL; if (dt == vect_external_def && TREE_CODE (oprnd1) == SSA_NAME @@ -2034,16 +1997,13 @@ vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_in, tree *type_out) Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the shift/rotate S3 stmt. */ static gimple * -vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts, - tree *type_in, tree *type_out) +vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); tree oprnd0, oprnd1, lhs, var; @@ -2085,9 +2045,8 @@ vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts, if (!def_vinfo) return NULL; - *type_in = get_vectype_for_scalar_type (TREE_TYPE (oprnd0)); - *type_out = *type_in; - if (*type_in == NULL_TREE) + *type_out = get_vectype_for_scalar_type (TREE_TYPE (oprnd0)); + if (*type_out == NULL_TREE) return NULL; tree def = NULL_TREE; @@ -2449,16 +2408,13 @@ vect_synth_mult_by_constant (tree op, tree val, Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the multiplication. */ static gimple * -vect_recog_mult_pattern (vec<gimple *> *stmts, - tree *type_in, tree *type_out) +vect_recog_mult_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); tree oprnd0, oprnd1, vectype, itype; @@ -2504,7 +2460,6 @@ vect_recog_mult_pattern (vec<gimple *> *stmts, vect_pattern_detected ("vect_recog_mult_pattern", last_stmt); stmts->safe_push (last_stmt); - *type_in = vectype; *type_out = vectype; return pattern_stmt; @@ -2543,16 +2498,13 @@ vect_recog_mult_pattern (vec<gimple *> *stmts, Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the division S1 or modulo S4 stmt. */ static gimple * -vect_recog_divmod_pattern (vec<gimple *> *stmts, - tree *type_in, tree *type_out) +vect_recog_divmod_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); tree oprnd0, oprnd1, vectype, itype, cond; @@ -2703,7 +2655,6 @@ vect_recog_divmod_pattern (vec<gimple *> *stmts, stmts->safe_push (last_stmt); - *type_in = vectype; *type_out = vectype; return pattern_stmt; } @@ -2955,7 +2906,6 @@ vect_recog_divmod_pattern (vec<gimple *> *stmts, stmts->safe_push (last_stmt); - *type_in = vectype; *type_out = vectype; return pattern_stmt; } @@ -2980,8 +2930,6 @@ vect_recog_divmod_pattern (vec<gimple *> *stmts, Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the pattern. @@ -2991,8 +2939,7 @@ vect_recog_divmod_pattern (vec<gimple *> *stmts, a_T = (TYPE) a_it; */ static gimple * -vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = (*stmts)[0]; tree cond_expr, then_clause, else_clause; @@ -3108,7 +3055,6 @@ vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_in, def_stmt_info = new_stmt_vec_info (def_stmt, vinfo); set_vinfo_for_stmt (def_stmt, def_stmt_info); STMT_VINFO_VECTYPE (def_stmt_info) = vecitype; - *type_in = vecitype; *type_out = vectype; vect_pattern_detected ("vect_recog_mixed_size_cond_pattern", last_stmt); @@ -3581,8 +3527,6 @@ search_type_for_mask (tree var, vec_info *vinfo) Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the pattern. @@ -3602,8 +3546,7 @@ search_type_for_mask (tree var, vec_info *vinfo) but the above is more efficient. */ static gimple * -vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); enum tree_code rhs_code; @@ -3679,7 +3622,6 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, } *type_out = vectype; - *type_in = vectype; stmts->safe_push (last_stmt); vect_pattern_detected ("vect_recog_bool_pattern", last_stmt); @@ -3718,7 +3660,6 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, gimple_assign_rhs2 (last_stmt), gimple_assign_rhs3 (last_stmt)); *type_out = vectype; - *type_in = vectype; stmts->safe_push (last_stmt); vect_pattern_detected ("vect_recog_bool_pattern", last_stmt); @@ -3775,7 +3716,6 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info) = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo); *type_out = vectype; - *type_in = vectype; stmts->safe_push (last_stmt); vect_pattern_detected ("vect_recog_bool_pattern", last_stmt); @@ -3839,8 +3779,7 @@ build_mask_conversion (tree mask, tree vectype, stmt_vec_info stmt_vinfo, S4' c_1' = m_3'' ? c_2 : c_3; */ static gimple * -vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); enum tree_code rhs_code; @@ -3911,7 +3850,6 @@ vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in, = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo); *type_out = vectype1; - *type_in = vectype1; stmts->safe_push (last_stmt); vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt); @@ -4035,7 +3973,6 @@ vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in, gimple_assign_rhs3 (last_stmt)); *type_out = vectype1; - *type_in = vectype1; stmts->safe_push (last_stmt); vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt); @@ -4081,7 +4018,6 @@ vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in, pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2); *type_out = vectype1; - *type_in = vectype1; stmts->safe_push (last_stmt); vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt); @@ -4180,14 +4116,14 @@ vect_add_conversion_to_patterm (tree type, tree value, /* Try to convert STMT into a call to a gather load or scatter store internal function. Return the final statement on success and set - *TYPE_IN and *TYPE_OUT to the vector type being loaded or stored. + *TYPE_OUT to the vector type being loaded or stored. This function only handles gathers and scatters that were recognized as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P). */ static gimple * vect_try_gather_scatter_pattern (gimple *stmt, stmt_vec_info last_stmt_info, - tree *type_in, tree *type_out) + tree *type_out) { /* Currently we only support this for loop vectorization. */ stmt_vec_info stmt_info = vinfo_for_stmt (stmt); @@ -4264,7 +4200,6 @@ vect_try_gather_scatter_pattern (gimple *stmt, stmt_vec_info last_stmt_info, tree vectype = STMT_VINFO_VECTYPE (stmt_info); *type_out = vectype; - *type_in = vectype; vect_pattern_detected ("gather/scatter pattern", stmt); return pattern_stmt; @@ -4273,19 +4208,52 @@ vect_try_gather_scatter_pattern (gimple *stmt, stmt_vec_info last_stmt_info, /* Pattern wrapper around vect_try_gather_scatter_pattern. */ static gimple * -vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); stmt_vec_info last_stmt_info = vinfo_for_stmt (last_stmt); gimple *pattern_stmt = vect_try_gather_scatter_pattern (last_stmt, last_stmt_info, - type_in, type_out); + type_out); if (pattern_stmt) stmts->safe_push (last_stmt); return pattern_stmt; } +typedef gimple *(*vect_recog_func_ptr) (vec<gimple *> *, tree *); + +struct vect_recog_func +{ + vect_recog_func_ptr fn; + const char *name; +}; + +/* Note that ordering matters - the first pattern matching on a stmt is + taken which means usually the more complex one needs to preceed the + less comples onex (widen_sum only after dot_prod or sad for example). */ +static vect_recog_func vect_vect_recog_func_ptrs[] = { + { vect_recog_widen_mult_pattern, "widen_mult" }, + { vect_recog_dot_prod_pattern, "dot_prod" }, + { vect_recog_sad_pattern, "sad" }, + { vect_recog_widen_sum_pattern, "widen_sum" }, + { vect_recog_pow_pattern, "pow" }, + { vect_recog_widen_shift_pattern, "widen_shift" }, + { vect_recog_over_widening_pattern, "over_widening" }, + { vect_recog_rotate_pattern, "rotate" }, + { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" }, + { vect_recog_divmod_pattern, "divmod" }, + { vect_recog_mult_pattern, "mult" }, + { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" }, + { vect_recog_bool_pattern, "bool" }, + /* This must come before mask conversion, and includes the parts + of mask conversion that are needed for gather and scatter + internal functions. */ + { vect_recog_gather_scatter_pattern, "gather_scatter" }, + { vect_recog_mask_conversion_pattern, "mask_conversion" } +}; + +const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs); + /* Mark statements that are involved in a pattern. */ static inline void @@ -4337,17 +4305,13 @@ vect_mark_pattern_stmts (gimple *orig_stmt, gimple *pattern_stmt, computation pattern. STMT: A stmt from which the pattern search should start. - If PATTERN_RECOG_FUNC successfully detected the pattern, it creates an - expression that computes the same functionality and can be used to - replace the sequence of stmts that are involved in the pattern. - - Output: - This function checks if the expression returned by PATTERN_RECOG_FUNC is - supported in vector form by the target. We use 'TYPE_IN' to obtain the - relevant vector type. If 'TYPE_IN' is already a vector type, then this - indicates that target support had already been checked by PATTERN_RECOG_FUNC. - If 'TYPE_OUT' is also returned by PATTERN_RECOG_FUNC, we check that it fits - to the available target pattern. + If PATTERN_RECOG_FUNC successfully detected the pattern, it creates + a sequence of statements that has the same functionality and can be + used to replace STMT. It returns the last statement in the sequence + and adds any earlier statements to STMT's STMT_VINFO_PATTERN_DEF_SEQ. + PATTERN_RECOG_FUNC also sets *TYPE_OUT to the vector type of the final + statement, having first checked that the target supports the new operation + in that type. This function also does some bookkeeping, as explained in the documentation for vect_recog_pattern. */ @@ -4361,13 +4325,11 @@ vect_pattern_recog_1 (vect_recog_func *recog_func, stmt_vec_info stmt_info; loop_vec_info loop_vinfo; tree pattern_vectype; - tree type_in, type_out; - enum tree_code code; int i; stmts_to_replace->truncate (0); stmts_to_replace->quick_push (stmt); - pattern_stmt = recog_func->fn (stmts_to_replace, &type_in, &type_out); + pattern_stmt = recog_func->fn (stmts_to_replace, &pattern_vectype); if (!pattern_stmt) { /* Clear related stmt info that analysis might have noted for @@ -4385,43 +4347,8 @@ vect_pattern_recog_1 (vect_recog_func *recog_func, stmt = stmts_to_replace->last (); stmt_info = vinfo_for_stmt (stmt); loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); + gcc_assert (pattern_vectype); - if (VECTOR_BOOLEAN_TYPE_P (type_in) - || VECTOR_TYPE_P (type_in)) - { - /* No need to check target support (already checked by the pattern - recognition function). */ - pattern_vectype = type_out ? type_out : type_in; - } - else - { - /* Check target support */ - type_in = get_vectype_for_scalar_type (type_in); - if (!type_in) - return false; - if (type_out) - type_out = get_vectype_for_scalar_type (type_out); - else - type_out = type_in; - if (!type_out) - return false; - pattern_vectype = type_out; - - if (is_gimple_assign (pattern_stmt)) - { - enum insn_code icode; - code = gimple_assign_rhs_code (pattern_stmt); - optab optab = optab_for_tree_code (code, type_in, optab_default); - machine_mode vec_mode = TYPE_MODE (type_in); - if (!optab - || (icode = optab_handler (optab, vec_mode)) == CODE_FOR_nothing - || (insn_data[icode].operand[0].mode != TYPE_MODE (type_out))) - return false; - } - else - gcc_assert (is_gimple_call (pattern_stmt)); - } - /* Found a vectorizable pattern. */ if (dump_enabled_p ()) { |