diff options
Diffstat (limited to 'gcc/tree-vect-patterns.c')
| -rw-r--r-- | gcc/tree-vect-patterns.c | 333 |
1 files changed, 280 insertions, 53 deletions
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index b575b45..e6c5bcd 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -48,6 +48,7 @@ along with GCC; see the file COPYING3. If not see #include "predict.h" #include "tree-vector-builder.h" #include "vec-perm-indices.h" +#include "gimple-range.h" /* Return true if we have a useful VR_RANGE range for VAR, storing it in *MIN_VALUE and *MAX_VALUE if so. Note the range in the dump files. */ @@ -55,7 +56,13 @@ along with GCC; see the file COPYING3. If not see static bool vect_get_range_info (tree var, wide_int *min_value, wide_int *max_value) { - value_range_kind vr_type = get_range_info (var, min_value, max_value); + value_range vr; + get_range_query (cfun)->range_of_expr (vr, var); + if (vr.undefined_p ()) + vr.set_varying (TREE_TYPE (var)); + *min_value = wi::to_wide (vr.min ()); + *max_value = wi::to_wide (vr.max ()); + value_range_kind vr_type = vr.kind (); wide_int nonzero = get_nonzero_bits (var); signop sgn = TYPE_SIGN (TREE_TYPE (var)); if (intersect_range_with_nonzero_bits (vr_type, min_value, max_value, @@ -184,9 +191,9 @@ vect_get_external_def_edge (vec_info *vinfo, tree var) } /* Return true if the target supports a vector version of CODE, - where CODE is known to map to a direct optab. ITYPE specifies - the type of (some of) the scalar inputs and OTYPE specifies the - type of the scalar result. + where CODE is known to map to a direct optab with the given SUBTYPE. + ITYPE specifies the type of (some of) the scalar inputs and OTYPE + specifies the type of the scalar result. If CODE allows the inputs and outputs to have different type (such as for WIDEN_SUM_EXPR), it is the input mode rather @@ -201,7 +208,8 @@ vect_get_external_def_edge (vec_info *vinfo, tree var) static bool vect_supportable_direct_optab_p (vec_info *vinfo, tree otype, tree_code code, tree itype, tree *vecotype_out, - tree *vecitype_out = NULL) + tree *vecitype_out = NULL, + enum optab_subtype subtype = optab_default) { tree vecitype = get_vectype_for_scalar_type (vinfo, itype); if (!vecitype) @@ -211,7 +219,7 @@ vect_supportable_direct_optab_p (vec_info *vinfo, tree otype, tree_code code, if (!vecotype) return false; - optab optab = optab_for_tree_code (code, vecitype, optab_default); + optab optab = optab_for_tree_code (code, vecitype, subtype); if (!optab) return false; @@ -514,6 +522,7 @@ vect_joust_widened_type (tree type, tree new_type, tree *common_type) unsigned int precision = MAX (TYPE_PRECISION (*common_type), TYPE_PRECISION (new_type)); precision *= 2; + if (precision * 2 > TYPE_PRECISION (type)) return false; @@ -532,6 +541,10 @@ vect_joust_widened_type (tree type, tree new_type, tree *common_type) to a type that (a) is narrower than the result of STMT_INFO and (b) can hold all leaf operand values. + If SUBTYPE then allow that the signs of the operands + may differ in signs but not in precision. SUBTYPE is updated to reflect + this. + Return 0 if STMT_INFO isn't such a tree, or if no such COMMON_TYPE exists. */ @@ -539,7 +552,8 @@ static unsigned int vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code, tree_code widened_code, bool shift_p, unsigned int max_nops, - vect_unpromoted_value *unprom, tree *common_type) + vect_unpromoted_value *unprom, tree *common_type, + enum optab_subtype *subtype = NULL) { /* Check for an integer operation with the right code. */ gassign *assign = dyn_cast <gassign *> (stmt_info->stmt); @@ -550,7 +564,7 @@ vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code, if (rhs_code != code && rhs_code != widened_code) return 0; - tree type = gimple_expr_type (assign); + tree type = TREE_TYPE (gimple_assign_lhs (assign)); if (!INTEGRAL_TYPE_P (type)) return 0; @@ -600,7 +614,8 @@ vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code, = vinfo->lookup_def (this_unprom->op); nops = vect_widened_op_tree (vinfo, def_stmt_info, code, widened_code, shift_p, max_nops, - this_unprom, common_type); + this_unprom, common_type, + subtype); if (nops == 0) return 0; @@ -618,7 +633,18 @@ vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code, *common_type = this_unprom->type; else if (!vect_joust_widened_type (type, this_unprom->type, common_type)) - return 0; + { + if (subtype) + { + /* See if we can sign extend the smaller type. */ + if (TYPE_PRECISION (this_unprom->type) + > TYPE_PRECISION (*common_type)) + *common_type = this_unprom->type; + *subtype = optab_vector_mixed_sign; + } + else + return 0; + } } } next_op += nops; @@ -718,12 +744,22 @@ vect_split_statement (vec_info *vinfo, stmt_vec_info stmt2_info, tree new_rhs, /* Convert UNPROM to TYPE and return the result, adding new statements to STMT_INFO's pattern definition statements if no better way is - available. VECTYPE is the vector form of TYPE. */ + available. VECTYPE is the vector form of TYPE. + + If SUBTYPE then convert the type based on the subtype. */ static tree vect_convert_input (vec_info *vinfo, stmt_vec_info stmt_info, tree type, - vect_unpromoted_value *unprom, tree vectype) + vect_unpromoted_value *unprom, tree vectype, + enum optab_subtype subtype = optab_default) { + + /* Update the type if the signs differ. */ + if (subtype == optab_vector_mixed_sign + && TYPE_SIGN (type) != TYPE_SIGN (TREE_TYPE (unprom->op))) + type = build_nonstandard_integer_type (TYPE_PRECISION (type), + TYPE_SIGN (unprom->type)); + /* Check for a no-op conversion. */ if (types_compatible_p (type, TREE_TYPE (unprom->op))) return unprom->op; @@ -799,12 +835,14 @@ vect_convert_input (vec_info *vinfo, stmt_vec_info stmt_info, tree type, } /* Invoke vect_convert_input for N elements of UNPROM and store the - result in the corresponding elements of RESULT. */ + result in the corresponding elements of RESULT. + + If SUBTYPE then convert the type based on the subtype. */ static void vect_convert_inputs (vec_info *vinfo, stmt_vec_info stmt_info, unsigned int n, tree *result, tree type, vect_unpromoted_value *unprom, - tree vectype) + tree vectype, enum optab_subtype subtype = optab_default) { for (unsigned int i = 0; i < n; ++i) { @@ -812,11 +850,12 @@ vect_convert_inputs (vec_info *vinfo, stmt_vec_info stmt_info, unsigned int n, for (j = 0; j < i; ++j) if (unprom[j].op == unprom[i].op) break; + if (j < i) result[i] = result[j]; else result[i] = vect_convert_input (vinfo, stmt_info, - type, &unprom[i], vectype); + type, &unprom[i], vectype, subtype); } } @@ -888,7 +927,8 @@ vect_reassociating_reduction_p (vec_info *vinfo, Try to find the following pattern: - type x_t, y_t; + type1a x_t + type1b y_t; TYPE1 prod; TYPE2 sum = init; loop: @@ -901,9 +941,9 @@ vect_reassociating_reduction_p (vec_info *vinfo, [S6 prod = (TYPE2) prod; #optional] S7 sum_1 = prod + sum_0; - where 'TYPE1' is exactly double the size of type 'type', and 'TYPE2' is the - same size of 'TYPE1' or bigger. This is a special case of a reduction - computation. + where 'TYPE1' is exactly double the size of type 'type1a' and 'type1b', + the sign of 'TYPE1' must be one of 'type1a' or 'type1b' but the sign of + 'type1a' and 'type1b' can differ. Input: @@ -946,7 +986,8 @@ vect_recog_dot_prod_pattern (vec_info *vinfo, In which - DX is double the size of X - DY is double the size of Y - - DX, DY, DPROD all have the same type + - DX, DY, DPROD all have the same type but the sign + between X, Y and DPROD can differ. - sum is the same size of DPROD or bigger - sum has been recognized as a reduction variable. @@ -965,7 +1006,7 @@ vect_recog_dot_prod_pattern (vec_info *vinfo, &oprnd0, &oprnd1)) return NULL; - type = gimple_expr_type (last_stmt); + type = TREE_TYPE (gimple_get_lhs (last_stmt)); vect_unpromoted_value unprom_mult; oprnd0 = vect_look_through_possible_promotion (vinfo, oprnd0, &unprom_mult); @@ -984,27 +1025,31 @@ vect_recog_dot_prod_pattern (vec_info *vinfo, /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi inside the loop (in case we are analyzing an outer-loop). */ vect_unpromoted_value unprom0[2]; + enum optab_subtype subtype = optab_vector; if (!vect_widened_op_tree (vinfo, mult_vinfo, MULT_EXPR, WIDEN_MULT_EXPR, - false, 2, unprom0, &half_type)) + false, 2, unprom0, &half_type, &subtype)) return NULL; - /* If there are two widening operations, make sure they agree on - the sign of the extension. */ + /* If there are two widening operations, make sure they agree on the sign + of the extension. The result of an optab_vector_mixed_sign operation + is signed; otherwise, the result has the same sign as the operands. */ if (TYPE_PRECISION (unprom_mult.type) != TYPE_PRECISION (type) - && TYPE_SIGN (unprom_mult.type) != TYPE_SIGN (half_type)) + && (subtype == optab_vector_mixed_sign + ? TYPE_UNSIGNED (unprom_mult.type) + : TYPE_SIGN (unprom_mult.type) != TYPE_SIGN (half_type))) return NULL; vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt); tree half_vectype; if (!vect_supportable_direct_optab_p (vinfo, type, DOT_PROD_EXPR, half_type, - type_out, &half_vectype)) + type_out, &half_vectype, subtype)) return NULL; /* Get the inputs in the appropriate types. */ tree mult_oprnd[2]; vect_convert_inputs (vinfo, stmt_vinfo, 2, mult_oprnd, half_type, - unprom0, half_vectype); + unprom0, half_vectype, subtype); var = vect_recog_temp_ssa_var (type, NULL); pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR, @@ -1090,7 +1135,7 @@ vect_recog_sad_pattern (vec_info *vinfo, &plus_oprnd0, &plus_oprnd1)) return NULL; - tree sum_type = gimple_expr_type (last_stmt); + tree sum_type = TREE_TYPE (gimple_get_lhs (last_stmt)); /* Any non-truncating sequence of conversions is OK here, since with a successful match, the result of the ABS(U) is known to fit @@ -1213,7 +1258,7 @@ vect_recog_widen_op_pattern (vec_info *vinfo, /* Pattern detected. */ vect_pattern_detected (name, last_stmt); - tree type = gimple_expr_type (last_stmt); + tree type = TREE_TYPE (gimple_get_lhs (last_stmt)); tree itype = type; if (TYPE_PRECISION (type) != TYPE_PRECISION (half_type) * 2 || TYPE_UNSIGNED (type) != TYPE_UNSIGNED (half_type)) @@ -1223,11 +1268,31 @@ vect_recog_widen_op_pattern (vec_info *vinfo, /* Check target support */ tree vectype = get_vectype_for_scalar_type (vinfo, half_type); tree vecitype = get_vectype_for_scalar_type (vinfo, itype); + tree ctype = itype; + tree vecctype = vecitype; + if (orig_code == MINUS_EXPR + && TYPE_UNSIGNED (itype) + && TYPE_PRECISION (type) > TYPE_PRECISION (itype)) + { + /* Subtraction is special, even if half_type is unsigned and no matter + whether type is signed or unsigned, if type is wider than itype, + we need to sign-extend from the widening operation result to the + result type. + Consider half_type unsigned char, operand 1 0xfe, operand 2 0xff, + itype unsigned short and type either int or unsigned int. + Widened (unsigned short) 0xfe - (unsigned short) 0xff is + (unsigned short) 0xffff, but for type int we want the result -1 + and for type unsigned int 0xffffffff rather than 0xffff. */ + ctype = build_nonstandard_integer_type (TYPE_PRECISION (itype), 0); + vecctype = get_vectype_for_scalar_type (vinfo, ctype); + } + enum tree_code dummy_code; int dummy_int; auto_vec<tree> dummy_vec; if (!vectype || !vecitype + || !vecctype || !supportable_widening_operation (vinfo, wide_code, last_stmt_info, vecitype, vectype, &dummy_code, &dummy_code, @@ -1246,8 +1311,12 @@ vect_recog_widen_op_pattern (vec_info *vinfo, gimple *pattern_stmt = gimple_build_assign (var, wide_code, oprnd[0], oprnd[1]); + if (vecctype != vecitype) + pattern_stmt = vect_convert_output (vinfo, last_stmt_info, ctype, + pattern_stmt, vecitype); + return vect_convert_output (vinfo, last_stmt_info, - type, pattern_stmt, vecitype); + type, pattern_stmt, vecctype); } /* Try to detect multiplication on widened inputs, converting MULT_EXPR @@ -1285,6 +1354,117 @@ vect_recog_widen_minus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info, "vect_recog_widen_minus_pattern"); } +/* Function vect_recog_popcount_pattern + + Try to find the following pattern: + + UTYPE1 A; + TYPE1 B; + UTYPE2 temp_in; + TYPE3 temp_out; + temp_in = (UTYPE2)A; + + temp_out = __builtin_popcount{,l,ll} (temp_in); + B = (TYPE1) temp_out; + + TYPE2 may or may not be equal to TYPE3. + i.e. TYPE2 is equal to TYPE3 for __builtin_popcount + i.e. TYPE2 is not equal to TYPE3 for __builtin_popcountll + + Input: + + * STMT_VINFO: The stmt from which the pattern search begins. + here it starts with B = (TYPE1) temp_out; + + Output: + + * TYPE_OUT: The vector type of the output of this pattern. + + * Return value: A new stmt that will be used to replace the sequence of + stmts that constitute the pattern. In this case it will be: + B = .POPCOUNT (A); +*/ + +static gimple * +vect_recog_popcount_pattern (vec_info *vinfo, + stmt_vec_info stmt_vinfo, tree *type_out) +{ + gassign *last_stmt = dyn_cast <gassign *> (stmt_vinfo->stmt); + gimple *popcount_stmt, *pattern_stmt; + tree rhs_oprnd, rhs_origin, lhs_oprnd, lhs_type, vec_type, new_var; + auto_vec<tree> vargs; + + /* Find B = (TYPE1) temp_out. */ + if (!last_stmt) + return NULL; + tree_code code = gimple_assign_rhs_code (last_stmt); + if (!CONVERT_EXPR_CODE_P (code)) + return NULL; + + lhs_oprnd = gimple_assign_lhs (last_stmt); + lhs_type = TREE_TYPE (lhs_oprnd); + if (!INTEGRAL_TYPE_P (lhs_type)) + return NULL; + + rhs_oprnd = gimple_assign_rhs1 (last_stmt); + if (TREE_CODE (rhs_oprnd) != SSA_NAME + || !has_single_use (rhs_oprnd)) + return NULL; + popcount_stmt = SSA_NAME_DEF_STMT (rhs_oprnd); + + /* Find temp_out = __builtin_popcount{,l,ll} (temp_in); */ + if (!is_gimple_call (popcount_stmt)) + return NULL; + switch (gimple_call_combined_fn (popcount_stmt)) + { + CASE_CFN_POPCOUNT: + break; + default: + return NULL; + } + + if (gimple_call_num_args (popcount_stmt) != 1) + return NULL; + + rhs_oprnd = gimple_call_arg (popcount_stmt, 0); + vect_unpromoted_value unprom_diff; + rhs_origin = vect_look_through_possible_promotion (vinfo, rhs_oprnd, + &unprom_diff); + + if (!rhs_origin) + return NULL; + + /* Input and output of .POPCOUNT should be same-precision integer. + Also A should be unsigned or same precision as temp_in, + otherwise there would be sign_extend from A to temp_in. */ + if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (lhs_type) + || (!TYPE_UNSIGNED (unprom_diff.type) + && (TYPE_PRECISION (unprom_diff.type) + != TYPE_PRECISION (TREE_TYPE (rhs_oprnd))))) + return NULL; + vargs.safe_push (unprom_diff.op); + + vect_pattern_detected ("vec_regcog_popcount_pattern", popcount_stmt); + vec_type = get_vectype_for_scalar_type (vinfo, lhs_type); + /* Do it only if the backend has popcount<vector_mode>2 pattern. */ + if (!vec_type + || !direct_internal_fn_supported_p (IFN_POPCOUNT, vec_type, + OPTIMIZE_FOR_SPEED)) + return NULL; + + /* Create B = .POPCOUNT (A). */ + new_var = vect_recog_temp_ssa_var (lhs_type, NULL); + pattern_stmt = gimple_build_call_internal_vec (IFN_POPCOUNT, vargs); + gimple_call_set_lhs (pattern_stmt, new_var); + gimple_set_location (pattern_stmt, gimple_location (last_stmt)); + *type_out = vec_type; + + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "created pattern stmt: %G", pattern_stmt); + return pattern_stmt; +} + /* Function vect_recog_pow_pattern Try to find the following pattern: @@ -1497,7 +1677,7 @@ vect_recog_widen_sum_pattern (vec_info *vinfo, &oprnd0, &oprnd1)) return NULL; - type = gimple_expr_type (last_stmt); + type = TREE_TYPE (gimple_get_lhs (last_stmt)); /* So far so good. Since last_stmt was detected as a (summation) reduction, we know that oprnd1 is the reduction variable (defined by a loop-header @@ -1705,6 +1885,7 @@ vect_recog_over_widening_pattern (vec_info *vinfo, /* Apply the minimum efficient precision we just calculated. */ if (new_precision < min_precision) new_precision = min_precision; + new_precision = vect_element_precision (new_precision); if (new_precision >= TYPE_PRECISION (type)) return NULL; @@ -1777,8 +1958,15 @@ vect_recog_over_widening_pattern (vec_info *vinfo, 1) Multiply high with scaling TYPE res = ((TYPE) a * (TYPE) b) >> c; + Here, c is bitsize (TYPE) / 2 - 1. + 2) ... or also with rounding TYPE res = (((TYPE) a * (TYPE) b) >> d + 1) >> 1; + Here, d is bitsize (TYPE) / 2 - 2. + + 3) Normal multiply high + TYPE res = ((TYPE) a * (TYPE) b) >> e; + Here, e is bitsize (TYPE) / 2. where only the bottom half of res is used. */ @@ -1822,8 +2010,7 @@ vect_recog_mulhs_pattern (vec_info *vinfo, stmt_vec_info mulh_stmt_info; tree scale_term; - internal_fn ifn; - unsigned int expect_offset; + bool rounding_p = false; /* Check for the presence of the rounding term. */ if (gimple_assign_rhs_code (rshift_input_stmt) == PLUS_EXPR) @@ -1872,23 +2059,16 @@ vect_recog_mulhs_pattern (vec_info *vinfo, /* Get the scaling term. */ scale_term = gimple_assign_rhs2 (plus_input_stmt); - - expect_offset = target_precision + 2; - ifn = IFN_MULHRS; + rounding_p = true; } else { mulh_stmt_info = rshift_input_stmt_info; scale_term = gimple_assign_rhs2 (last_stmt); - - expect_offset = target_precision + 1; - ifn = IFN_MULHS; } - /* Check that the scaling factor is correct. */ - if (TREE_CODE (scale_term) != INTEGER_CST - || wi::to_widest (scale_term) + expect_offset - != TYPE_PRECISION (lhs_type)) + /* Check that the scaling factor is constant. */ + if (TREE_CODE (scale_term) != INTEGER_CST) return NULL; /* Check whether the scaling input term can be seen as two widened @@ -1901,13 +2081,41 @@ vect_recog_mulhs_pattern (vec_info *vinfo, if (nops != 2) return NULL; - vect_pattern_detected ("vect_recog_mulhs_pattern", last_stmt); - /* Adjust output precision. */ if (TYPE_PRECISION (new_type) < target_precision) new_type = build_nonstandard_integer_type (target_precision, TYPE_UNSIGNED (new_type)); + unsigned mult_precision = TYPE_PRECISION (new_type); + internal_fn ifn; + /* Check that the scaling factor is expected. Instead of + target_precision, we should use the one that we actually + use for internal function. */ + if (rounding_p) + { + /* Check pattern 2). */ + if (wi::to_widest (scale_term) + mult_precision + 2 + != TYPE_PRECISION (lhs_type)) + return NULL; + + ifn = IFN_MULHRS; + } + else + { + /* Check for pattern 1). */ + if (wi::to_widest (scale_term) + mult_precision + 1 + == TYPE_PRECISION (lhs_type)) + ifn = IFN_MULHS; + /* Check for pattern 3). */ + else if (wi::to_widest (scale_term) + mult_precision + == TYPE_PRECISION (lhs_type)) + ifn = IFN_MULH; + else + return NULL; + } + + vect_pattern_detected ("vect_recog_mulhs_pattern", last_stmt); + /* Check for target support. */ tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type); if (!new_vectype @@ -3436,13 +3644,14 @@ vect_recog_divmod_pattern (vec_info *vinfo, else t3 = t2; - wide_int oprnd0_min, oprnd0_max; int msb = 1; - if (get_range_info (oprnd0, &oprnd0_min, &oprnd0_max) == VR_RANGE) + value_range r; + get_range_query (cfun)->range_of_expr (r, oprnd0); + if (r.kind () == VR_RANGE) { - if (!wi::neg_p (oprnd0_min, TYPE_SIGN (itype))) + if (!wi::neg_p (r.lower_bound (), TYPE_SIGN (itype))) msb = 0; - else if (wi::neg_p (oprnd0_max, TYPE_SIGN (itype))) + else if (wi::neg_p (r.upper_bound (), TYPE_SIGN (itype))) msb = -1; } @@ -3558,7 +3767,7 @@ vect_recog_mixed_size_cond_pattern (vec_info *vinfo, if (comp_vectype == NULL_TREE) return NULL; - type = gimple_expr_type (last_stmt); + type = TREE_TYPE (gimple_assign_lhs (last_stmt)); if (types_compatible_p (type, comp_scalar_type) || ((TREE_CODE (then_clause) != INTEGER_CST || TREE_CODE (else_clause) != INTEGER_CST) @@ -4626,7 +4835,7 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo, function for the gather/scatter operation. */ gather_scatter_info gs_info; if (!vect_check_gather_scatter (stmt_info, loop_vinfo, &gs_info) - || gs_info.decl) + || gs_info.ifn == IFN_LAST) return NULL; /* Convert the mask to the right form. */ @@ -4635,6 +4844,9 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo, if (mask) mask = vect_convert_mask_for_vectype (mask, gs_vectype, stmt_info, loop_vinfo); + else if (gs_info.ifn == IFN_MASK_SCATTER_STORE + || gs_info.ifn == IFN_MASK_GATHER_LOAD) + mask = build_int_cst (TREE_TYPE (truth_type_for (gs_vectype)), -1); /* Get the invariant base and non-invariant offset, converting the latter to the same width as the vector elements. */ @@ -4662,11 +4874,11 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo, { tree rhs = vect_get_store_rhs (stmt_info); if (mask != NULL) - pattern_stmt = gimple_build_call_internal (IFN_MASK_SCATTER_STORE, 5, + pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5, base, offset, scale, rhs, mask); else - pattern_stmt = gimple_build_call_internal (IFN_SCATTER_STORE, 4, + pattern_stmt = gimple_build_call_internal (gs_info.ifn, 4, base, offset, scale, rhs); } gimple_call_set_nothrow (pattern_stmt, true); @@ -5197,6 +5409,13 @@ vect_determine_precisions (vec_info *vinfo) for (unsigned int i = 0; i < nbbs; i++) { basic_block bb = bbs[i]; + for (auto gsi = gsi_start_phis (bb); + !gsi_end_p (gsi); gsi_next (&gsi)) + { + stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi.phi ()); + if (stmt_info) + vect_determine_mask_precision (vinfo, stmt_info); + } for (auto si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) if (!is_gimple_debug (gsi_stmt (si))) vect_determine_mask_precision @@ -5210,6 +5429,13 @@ vect_determine_precisions (vec_info *vinfo) if (!is_gimple_debug (gsi_stmt (si))) vect_determine_stmt_precisions (vinfo, vinfo->lookup_stmt (gsi_stmt (si))); + for (auto gsi = gsi_start_phis (bb); + !gsi_end_p (gsi); gsi_next (&gsi)) + { + stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi.phi ()); + if (stmt_info) + vect_determine_stmt_precisions (vinfo, stmt_info); + } } } else @@ -5274,6 +5500,7 @@ static vect_recog_func vect_vect_recog_func_ptrs[] = { { vect_recog_sad_pattern, "sad" }, { vect_recog_widen_sum_pattern, "widen_sum" }, { vect_recog_pow_pattern, "pow" }, + { vect_recog_popcount_pattern, "popcount" }, { vect_recog_widen_shift_pattern, "widen_shift" }, { vect_recog_rotate_pattern, "rotate" }, { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" }, |