diff options
Diffstat (limited to 'gcc/tree-vect-data-refs.c')
| -rw-r--r-- | gcc/tree-vect-data-refs.c | 265 |
1 files changed, 151 insertions, 114 deletions
diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 97c8577..f43d0f4 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -53,6 +53,7 @@ along with GCC; see the file COPYING3. If not see #include "tree-hash-traits.h" #include "vec-perm-indices.h" #include "internal-fn.h" +#include "gimple-fold.h" /* Return true if load- or store-lanes optab OPTAB is implemented for COUNT vectors of type VECTYPE. NAME is the name of OPTAB. */ @@ -115,11 +116,8 @@ vect_lanes_optab_supported_p (const char *name, convert_optab optab, types. */ tree -vect_get_smallest_scalar_type (stmt_vec_info stmt_info, - HOST_WIDE_INT *lhs_size_unit, - HOST_WIDE_INT *rhs_size_unit) +vect_get_smallest_scalar_type (stmt_vec_info stmt_info, tree scalar_type) { - tree scalar_type = gimple_expr_type (stmt_info->stmt); HOST_WIDE_INT lhs, rhs; /* During the analysis phase, this function is called on arbitrary @@ -130,21 +128,24 @@ vect_get_smallest_scalar_type (stmt_vec_info stmt_info, lhs = rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type)); gassign *assign = dyn_cast <gassign *> (stmt_info->stmt); - if (assign - && (gimple_assign_cast_p (assign) + if (assign) + { + scalar_type = TREE_TYPE (gimple_assign_lhs (assign)); + if (gimple_assign_cast_p (assign) || gimple_assign_rhs_code (assign) == DOT_PROD_EXPR || gimple_assign_rhs_code (assign) == WIDEN_SUM_EXPR || gimple_assign_rhs_code (assign) == WIDEN_MULT_EXPR || gimple_assign_rhs_code (assign) == WIDEN_LSHIFT_EXPR || gimple_assign_rhs_code (assign) == WIDEN_PLUS_EXPR || gimple_assign_rhs_code (assign) == WIDEN_MINUS_EXPR - || gimple_assign_rhs_code (assign) == FLOAT_EXPR)) - { - tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (assign)); + || gimple_assign_rhs_code (assign) == FLOAT_EXPR) + { + tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (assign)); - rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (rhs_type)); - if (rhs < lhs) - scalar_type = rhs_type; + rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (rhs_type)); + if (rhs < lhs) + scalar_type = rhs_type; + } } else if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt)) { @@ -152,10 +153,16 @@ vect_get_smallest_scalar_type (stmt_vec_info stmt_info, if (gimple_call_internal_p (call)) { internal_fn ifn = gimple_call_internal_fn (call); - if (internal_load_fn_p (ifn) || internal_store_fn_p (ifn)) - /* gimple_expr_type already picked the type of the loaded - or stored data. */ + if (internal_load_fn_p (ifn)) + /* For loads the LHS type does the trick. */ i = ~0U; + else if (internal_store_fn_p (ifn)) + { + /* For stores use the tyep of the stored value. */ + i = internal_fn_stored_value_index (ifn); + scalar_type = TREE_TYPE (gimple_call_arg (call, i)); + i = ~0U; + } else if (internal_fn_mask_index (ifn) == 0) i = 1; } @@ -171,8 +178,6 @@ vect_get_smallest_scalar_type (stmt_vec_info stmt_info, } } - *lhs_size_unit = lhs; - *rhs_size_unit = rhs; return scalar_type; } @@ -207,7 +212,7 @@ vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo) static void vect_check_nonzero_value (loop_vec_info loop_vinfo, tree value) { - vec<tree> checks = LOOP_VINFO_CHECK_NONZERO (loop_vinfo); + const vec<tree> &checks = LOOP_VINFO_CHECK_NONZERO (loop_vinfo); for (unsigned int i = 0; i < checks.length(); ++i) if (checks[i] == value) return; @@ -290,9 +295,7 @@ vect_analyze_possibly_independent_ddr (data_dependence_relation *ddr, int loop_depth, unsigned int *max_vf) { class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); - lambda_vector dist_v; - unsigned int i; - FOR_EACH_VEC_ELT (DDR_DIST_VECTS (ddr), i, dist_v) + for (lambda_vector &dist_v : DDR_DIST_VECTS (ddr)) { int dist = dist_v[loop_depth]; if (dist != 0 && !(dist > 0 && DDR_REVERSED_P (ddr))) @@ -729,9 +732,8 @@ vect_slp_analyze_node_dependences (vec_info *vinfo, slp_tree node, { if (stmt_info != last_store_info) continue; - unsigned i; - stmt_vec_info store_info; - FOR_EACH_VEC_ELT (stores, i, store_info) + + for (stmt_vec_info &store_info : stores) { data_reference *store_dr = STMT_VINFO_DATA_REF (store_info); @@ -780,16 +782,20 @@ vect_slp_analyze_node_dependences (vec_info *vinfo, slp_tree node, stmt we have to resort to the alias oracle. */ stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt); data_reference *dr_b = STMT_VINFO_DATA_REF (stmt_info); - if (!dr_b) + + /* We are hoisting a load - this means we can use + TBAA for disambiguation. */ + if (!ref_initialized_p) + ao_ref_init (&ref, DR_REF (dr_a)); + if (stmt_may_clobber_ref_p_1 (stmt, &ref, true)) { - /* We are hoisting a load - this means we can use - TBAA for disambiguation. */ - if (!ref_initialized_p) - ao_ref_init (&ref, DR_REF (dr_a)); - if (stmt_may_clobber_ref_p_1 (stmt, &ref, true)) + if (!dr_b) return false; - continue; + /* Resort to dependence checking below. */ } + else + /* No dependence. */ + continue; bool dependent = false; /* If we run into a store of this same instance (we've just @@ -800,9 +806,8 @@ vect_slp_analyze_node_dependences (vec_info *vinfo, slp_tree node, { if (stmt_info != last_store_info) continue; - unsigned i; - stmt_vec_info store_info; - FOR_EACH_VEC_ELT (stores, i, store_info) + + for (stmt_vec_info &store_info : stores) { data_reference *store_dr = STMT_VINFO_DATA_REF (store_info); @@ -843,9 +848,9 @@ vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance) DUMP_VECT_SCOPE ("vect_slp_analyze_instance_dependence"); /* The stores of this instance are at the root of the SLP tree. */ - slp_tree store = SLP_INSTANCE_TREE (instance); - if (! STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (store))) - store = NULL; + slp_tree store = NULL; + if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store) + store = SLP_INSTANCE_TREE (instance); /* Verify we can sink stores to the vectorized stmt insert location. */ stmt_vec_info last_store_info = NULL; @@ -864,9 +869,7 @@ vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance) /* Verify we can sink loads to the vectorized stmt insert location, special-casing stores of this instance. */ - slp_tree load; - unsigned int i; - FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, load) + for (slp_tree &load : SLP_INSTANCE_LOADS (instance)) if (! vect_slp_analyze_node_dependences (vinfo, load, store ? SLP_TREE_SCALAR_STMTS (store) @@ -892,11 +895,11 @@ vect_record_base_alignment (vec_info *vinfo, stmt_vec_info stmt_info, innermost_loop_behavior *drb) { bool existed; - innermost_loop_behavior *&entry + std::pair<stmt_vec_info, innermost_loop_behavior *> &entry = vinfo->base_alignments.get_or_insert (drb->base_address, &existed); - if (!existed || entry->base_alignment < drb->base_alignment) + if (!existed || entry.second->base_alignment < drb->base_alignment) { - entry = drb; + entry = std::make_pair (stmt_info, drb); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "recording new base alignment for %T\n" @@ -923,9 +926,7 @@ vect_record_base_alignments (vec_info *vinfo) { loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL; - data_reference *dr; - unsigned int i; - FOR_EACH_VEC_ELT (vinfo->shared->datarefs, i, dr) + for (data_reference *dr : vinfo->shared->datarefs) { dr_vec_info *dr_info = vinfo->lookup_dr (dr); stmt_vec_info stmt_info = dr_info->stmt; @@ -1059,11 +1060,18 @@ vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info) /* Calculate the maximum of the pooled base address alignment and the alignment that we can compute for DR itself. */ - innermost_loop_behavior **entry = base_alignments->get (drb->base_address); - if (entry && base_alignment < (*entry)->base_alignment) + std::pair<stmt_vec_info, innermost_loop_behavior *> *entry + = base_alignments->get (drb->base_address); + if (entry + && base_alignment < (*entry).second->base_alignment + && (loop_vinfo + || (dominated_by_p (CDI_DOMINATORS, gimple_bb (stmt_info->stmt), + gimple_bb (entry->first->stmt)) + && (gimple_bb (stmt_info->stmt) != gimple_bb (entry->first->stmt) + || (entry->first->dr_aux.group <= dr_info->group))))) { - base_alignment = (*entry)->base_alignment; - base_misalignment = (*entry)->base_misalignment; + base_alignment = entry->second->base_alignment; + base_misalignment = entry->second->base_misalignment; } if (drb->offset_alignment < vect_align_c @@ -1459,10 +1467,8 @@ vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo, bool unknown_misalignment) { vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo); - unsigned i; - data_reference *dr; - FOR_EACH_VEC_ELT (datarefs, i, dr) + for (data_reference *dr : datarefs) { dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr); if (!vect_relevant_for_alignment_p (dr_info)) @@ -1571,13 +1577,11 @@ static bool vect_peeling_supportable (loop_vec_info loop_vinfo, dr_vec_info *dr0_info, unsigned npeel) { - unsigned i; - struct data_reference *dr = NULL; vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo); enum dr_alignment_support supportable_dr_alignment; /* Ensure that all data refs can be vectorized after the peel. */ - FOR_EACH_VEC_ELT (datarefs, i, dr) + for (data_reference *dr : datarefs) { int save_misalignment; @@ -2352,8 +2356,8 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) if (do_versioning) { - vec<stmt_vec_info> may_misalign_stmts - = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo); + const vec<stmt_vec_info> &may_misalign_stmts + = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo); stmt_vec_info stmt_info; /* It can now be assumed that the data references in the statements @@ -2472,8 +2476,7 @@ vect_slp_analyze_instance_alignment (vec_info *vinfo, if (! vect_slp_analyze_node_alignment (vinfo, node)) return false; - node = SLP_INSTANCE_TREE (instance); - if (STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (node)) + if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store && ! vect_slp_analyze_node_alignment (vinfo, SLP_INSTANCE_TREE (instance))) return false; @@ -2817,33 +2820,25 @@ vect_analyze_data_ref_access (vec_info *vinfo, dr_vec_info *dr_info) return vect_analyze_group_access (vinfo, dr_info); } -typedef std::pair<data_reference_p, int> data_ref_pair; - /* Compare two data-references DRA and DRB to group them into chunks suitable for grouping. */ static int dr_group_sort_cmp (const void *dra_, const void *drb_) { - data_ref_pair dra_pair = *(data_ref_pair *)const_cast<void *>(dra_); - data_ref_pair drb_pair = *(data_ref_pair *)const_cast<void *>(drb_); - data_reference_p dra = dra_pair.first; - data_reference_p drb = drb_pair.first; + dr_vec_info *dra_info = *(dr_vec_info **)const_cast<void *>(dra_); + dr_vec_info *drb_info = *(dr_vec_info **)const_cast<void *>(drb_); + data_reference_p dra = dra_info->dr; + data_reference_p drb = drb_info->dr; int cmp; /* Stabilize sort. */ if (dra == drb) return 0; - /* DRs in different basic-blocks never belong to the same group. */ - int bb_index1 = gimple_bb (DR_STMT (dra))->index; - int bb_index2 = gimple_bb (DR_STMT (drb))->index; - if (bb_index1 != bb_index2) - return bb_index1 < bb_index2 ? -1 : 1; - /* Different group IDs lead never belong to the same group. */ - if (dra_pair.second != drb_pair.second) - return dra_pair.second < drb_pair.second ? -1 : 1; + if (dra_info->group != drb_info->group) + return dra_info->group < drb_info->group ? -1 : 1; /* Ordering of DRs according to base. */ cmp = data_ref_compare_tree (DR_BASE_ADDRESS (dra), @@ -2963,12 +2958,18 @@ vect_analyze_data_ref_accesses (vec_info *vinfo, /* Sort the array of datarefs to make building the interleaving chains linear. Don't modify the original vector's order, it is needed for determining what dependencies are reversed. */ - vec<data_ref_pair> datarefs_copy; + vec<dr_vec_info *> datarefs_copy; datarefs_copy.create (datarefs.length ()); for (unsigned i = 0; i < datarefs.length (); i++) { - int group_id = dataref_groups ? (*dataref_groups)[i] : 0; - datarefs_copy.quick_push (data_ref_pair (datarefs[i], group_id)); + dr_vec_info *dr_info = vinfo->lookup_dr (datarefs[i]); + /* If the caller computed DR grouping use that, otherwise group by + basic blocks. */ + if (dataref_groups) + dr_info->group = (*dataref_groups)[i]; + else + dr_info->group = gimple_bb (DR_STMT (datarefs[i]))->index; + datarefs_copy.quick_push (dr_info); } datarefs_copy.qsort (dr_group_sort_cmp); hash_set<stmt_vec_info> to_fixup; @@ -2976,9 +2977,9 @@ vect_analyze_data_ref_accesses (vec_info *vinfo, /* Build the interleaving chains. */ for (i = 0; i < datarefs_copy.length () - 1;) { - data_reference_p dra = datarefs_copy[i].first; - int dra_group_id = datarefs_copy[i].second; - dr_vec_info *dr_info_a = vinfo->lookup_dr (dra); + dr_vec_info *dr_info_a = datarefs_copy[i]; + data_reference_p dra = dr_info_a->dr; + int dra_group_id = dr_info_a->group; stmt_vec_info stmtinfo_a = dr_info_a->stmt; stmt_vec_info lastinfo = NULL; if (!STMT_VINFO_VECTORIZABLE (stmtinfo_a) @@ -2989,9 +2990,9 @@ vect_analyze_data_ref_accesses (vec_info *vinfo, } for (i = i + 1; i < datarefs_copy.length (); ++i) { - data_reference_p drb = datarefs_copy[i].first; - int drb_group_id = datarefs_copy[i].second; - dr_vec_info *dr_info_b = vinfo->lookup_dr (drb); + dr_vec_info *dr_info_b = datarefs_copy[i]; + data_reference_p drb = dr_info_b->dr; + int drb_group_id = dr_info_b->group; stmt_vec_info stmtinfo_b = dr_info_b->stmt; if (!STMT_VINFO_VECTORIZABLE (stmtinfo_b) || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b)) @@ -3003,13 +3004,8 @@ vect_analyze_data_ref_accesses (vec_info *vinfo, matters we can push those to a worklist and re-iterate over them. The we can just skip ahead to the next DR here. */ - /* DRs in a different BBs should not be put into the same + /* DRs in a different DR group should not be put into the same interleaving group. */ - int bb_index1 = gimple_bb (DR_STMT (dra))->index; - int bb_index2 = gimple_bb (DR_STMT (drb))->index; - if (bb_index1 != bb_index2) - break; - if (dra_group_id != drb_group_id) break; @@ -3057,7 +3053,7 @@ vect_analyze_data_ref_accesses (vec_info *vinfo, HOST_WIDE_INT init_a = TREE_INT_CST_LOW (DR_INIT (dra)); HOST_WIDE_INT init_b = TREE_INT_CST_LOW (DR_INIT (drb)); HOST_WIDE_INT init_prev - = TREE_INT_CST_LOW (DR_INIT (datarefs_copy[i-1].first)); + = TREE_INT_CST_LOW (DR_INIT (datarefs_copy[i-1]->dr)); gcc_assert (init_a <= init_b && init_a <= init_prev && init_prev <= init_b); @@ -3065,7 +3061,7 @@ vect_analyze_data_ref_accesses (vec_info *vinfo, /* Do not place the same access in the interleaving chain twice. */ if (init_b == init_prev) { - gcc_assert (gimple_uid (DR_STMT (datarefs_copy[i-1].first)) + gcc_assert (gimple_uid (DR_STMT (datarefs_copy[i-1]->dr)) < gimple_uid (DR_STMT (drb))); /* Simply link in duplicates and fix up the chain below. */ } @@ -3178,10 +3174,9 @@ vect_analyze_data_ref_accesses (vec_info *vinfo, to_fixup.add (newgroup); } - data_ref_pair *dr_pair; - FOR_EACH_VEC_ELT (datarefs_copy, i, dr_pair) + dr_vec_info *dr_info; + FOR_EACH_VEC_ELT (datarefs_copy, i, dr_info) { - dr_vec_info *dr_info = vinfo->lookup_dr (dr_pair->first); if (STMT_VINFO_VECTORIZABLE (dr_info->stmt) && !vect_analyze_data_ref_access (vinfo, dr_info)) { @@ -3368,7 +3363,8 @@ static void vect_check_lower_bound (loop_vec_info loop_vinfo, tree expr, bool unsigned_p, poly_uint64 min_value) { - vec<vec_lower_bound> lower_bounds = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo); + vec<vec_lower_bound> &lower_bounds + = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo); for (unsigned int i = 0; i < lower_bounds.length (); ++i) if (operand_equal_p (lower_bounds[i].expr, expr, 0)) { @@ -3470,10 +3466,10 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo) typedef pair_hash <tree_operand_hash, tree_operand_hash> tree_pair_hash; hash_set <tree_pair_hash> compared_objects; - vec<ddr_p> may_alias_ddrs = LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo); + const vec<ddr_p> &may_alias_ddrs = LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo); vec<dr_with_seg_len_pair_t> &comp_alias_ddrs = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo); - vec<vec_object_pair> &check_unequal_addrs + const vec<vec_object_pair> &check_unequal_addrs = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo); poly_uint64 vect_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo); tree scalar_loop_iters = LOOP_VINFO_NITERS (loop_vinfo); @@ -3487,9 +3483,9 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo) /* Step values are irrelevant for aliasing if the number of vector iterations is equal to the number of scalar iterations (which can happen for fully-SLP loops). */ - bool ignore_step_p = known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U); + bool vf_one_p = known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U); - if (!ignore_step_p) + if (!vf_one_p) { /* Convert the checks for nonzero steps into bound tests. */ tree value; @@ -3542,6 +3538,11 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo) bool preserves_scalar_order_p = vect_preserves_scalar_order_p (dr_info_a, dr_info_b); + bool ignore_step_p + = (vf_one_p + && (preserves_scalar_order_p + || operand_equal_p (DR_STEP (dr_info_a->dr), + DR_STEP (dr_info_b->dr)))); /* Skip the pair if inter-iteration dependencies are irrelevant and intra-iteration dependencies are guaranteed to be honored. */ @@ -3733,11 +3734,17 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p, return false; /* Work out which function we need. */ - internal_fn ifn; + internal_fn ifn, alt_ifn; if (read_p) - ifn = masked_p ? IFN_MASK_GATHER_LOAD : IFN_GATHER_LOAD; + { + ifn = masked_p ? IFN_MASK_GATHER_LOAD : IFN_GATHER_LOAD; + alt_ifn = IFN_MASK_GATHER_LOAD; + } else - ifn = masked_p ? IFN_MASK_SCATTER_STORE : IFN_SCATTER_STORE; + { + ifn = masked_p ? IFN_MASK_SCATTER_STORE : IFN_SCATTER_STORE; + alt_ifn = IFN_MASK_SCATTER_STORE; + } for (;;) { @@ -3753,6 +3760,16 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p, *offset_vectype_out = offset_vectype; return true; } + else if (!masked_p + && internal_gather_scatter_fn_supported_p (alt_ifn, vectype, + memory_type, + offset_vectype, + scale)) + { + *ifn_out = alt_ifn; + *offset_vectype_out = offset_vectype; + return true; + } if (TYPE_PRECISION (offset_type) >= POINTER_SIZE && TYPE_PRECISION (offset_type) >= element_bits) @@ -4006,8 +4023,24 @@ vect_check_gather_scatter (stmt_vec_info stmt_info, loop_vec_info loop_vinfo, continue; } - if (TYPE_PRECISION (TREE_TYPE (op0)) - < TYPE_PRECISION (TREE_TYPE (off))) + /* Include the conversion if it is widening and we're using + the IFN path or the target can handle the converted from + offset or the current size is not already the same as the + data vector element size. */ + if ((TYPE_PRECISION (TREE_TYPE (op0)) + < TYPE_PRECISION (TREE_TYPE (off))) + && (use_ifn_p + || (DR_IS_READ (dr) + ? (targetm.vectorize.builtin_gather + && targetm.vectorize.builtin_gather (vectype, + TREE_TYPE (op0), + scale)) + : (targetm.vectorize.builtin_scatter + && targetm.vectorize.builtin_scatter (vectype, + TREE_TYPE (op0), + scale))) + || !operand_equal_p (TYPE_SIZE (TREE_TYPE (off)), + TYPE_SIZE (TREE_TYPE (vectype)), 0))) { off = op0; offtype = TREE_TYPE (off); @@ -4035,7 +4068,8 @@ vect_check_gather_scatter (stmt_vec_info stmt_info, loop_vec_info loop_vinfo, if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p, vectype, memory_type, offtype, scale, &ifn, &offset_vectype)) - return false; + ifn = IFN_LAST; + decl = NULL_TREE; } else { @@ -4049,10 +4083,6 @@ vect_check_gather_scatter (stmt_vec_info stmt_info, loop_vec_info loop_vinfo, if (targetm.vectorize.builtin_scatter) decl = targetm.vectorize.builtin_scatter (vectype, offtype, scale); } - - if (!decl) - return false; - ifn = IFN_LAST; /* The offset vector type will be read from DECL when needed. */ offset_vectype = NULL_TREE; @@ -4282,9 +4312,7 @@ vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf, bool *fatal) { bool maybe_gather = DR_IS_READ (dr) - && !TREE_THIS_VOLATILE (DR_REF (dr)) - && (targetm.vectorize.builtin_gather != NULL - || supports_vec_gather_load_p ()); + && !TREE_THIS_VOLATILE (DR_REF (dr)); bool maybe_scatter = DR_IS_WRITE (dr) && !TREE_THIS_VOLATILE (DR_REF (dr)) @@ -5035,7 +5063,7 @@ bump_vector_ptr (vec_info *vinfo, struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); tree vectype = STMT_VINFO_VECTYPE (stmt_info); tree update = TYPE_SIZE_UNIT (vectype); - gassign *incr_stmt; + gimple *incr_stmt; ssa_op_iter iter; use_operand_p use_p; tree new_dataref_ptr; @@ -5050,6 +5078,15 @@ bump_vector_ptr (vec_info *vinfo, incr_stmt = gimple_build_assign (new_dataref_ptr, POINTER_PLUS_EXPR, dataref_ptr, update); vect_finish_stmt_generation (vinfo, stmt_info, incr_stmt, gsi); + /* Fold the increment, avoiding excessive chains use-def chains of + those, leading to compile-time issues for passes until the next + forwprop pass which would do this as well. */ + gimple_stmt_iterator fold_gsi = gsi_for_stmt (incr_stmt); + if (fold_stmt (&fold_gsi, follow_all_ssa_edges)) + { + incr_stmt = gsi_stmt (fold_gsi); + update_stmt (incr_stmt); + } /* Copy the points-to information if it exists. */ if (DR_PTR_INFO (dr)) @@ -5329,7 +5366,7 @@ vect_store_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count, I4: 6 14 22 30 7 15 23 31. */ void -vect_permute_store_chain (vec_info *vinfo, vec<tree> dr_chain, +vect_permute_store_chain (vec_info *vinfo, vec<tree> &dr_chain, unsigned int length, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, |