diff options
Diffstat (limited to 'gcc/tree-vect-stmts.cc')
| -rw-r--r-- | gcc/tree-vect-stmts.cc | 8216 |
1 files changed, 3601 insertions, 4615 deletions
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc index 38612a1..56ff1c8 100644 --- a/gcc/tree-vect-stmts.cc +++ b/gcc/tree-vect-stmts.cc @@ -62,13 +62,8 @@ along with GCC; see the file COPYING3. If not see /* For lang_hooks.types.type_for_mode. */ #include "langhooks.h" -/* Return the vectorized type for the given statement. */ - -tree -stmt_vectype (class _stmt_vec_info *stmt_info) -{ - return STMT_VINFO_VECTYPE (stmt_info); -} +static tree vector_vector_composition_type (tree, poly_uint64, tree *, + bool = false); /* Return TRUE iff the given statement is in an inner loop relative to the loop being vectorized. */ @@ -130,7 +125,8 @@ record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count, tree vectype, int misalign, enum vect_cost_model_location where) { - return record_stmt_cost (body_cost_vec, count, kind, NULL, node, + return record_stmt_cost (body_cost_vec, count, kind, + SLP_TREE_REPRESENTATIVE (node), node, vectype, misalign, where); } @@ -360,7 +356,6 @@ is_simple_and_all_uses_invariant (stmt_vec_info stmt_info, - it has uses outside the loop. - it has vdefs (it alters memory). - control stmts in the loop (except for the exit condition). - - it is an induction and we have multiple exits. CHECKME: what other side effects would the vectorizer allow? */ @@ -393,6 +388,9 @@ vect_stmt_relevant_p (stmt_vec_info stmt_info, loop_vec_info loop_vinfo, dump_printf_loc (MSG_NOTE, vect_location, "vec_stmt_relevant_p: stmt has vdefs.\n"); *relevant = vect_used_in_scope; + if (! STMT_VINFO_DATA_REF (stmt_info) + && zero_ssa_operands (stmt_info->stmt, SSA_OP_DEF)) + LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo).safe_push (stmt_info); } /* uses outside the loop. */ @@ -419,20 +417,6 @@ vect_stmt_relevant_p (stmt_vec_info stmt_info, loop_vec_info loop_vinfo, } } - /* Check if it's an induction and multiple exits. In this case there will be - a usage later on after peeling which is needed for the alternate exit. */ - if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo) - && STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "vec_stmt_relevant_p: induction forced for " - "early break.\n"); - LOOP_VINFO_EARLY_BREAKS_LIVE_IVS (loop_vinfo).safe_push (stmt_info); - *live_p = true; - - } - if (*live_p && *relevant == vect_unused_in_scope && !is_simple_and_all_uses_invariant (stmt_info, loop_vinfo)) { @@ -655,14 +639,15 @@ process_use (stmt_vec_info stmt_vinfo, tree use, loop_vec_info loop_vinfo, } /* We are also not interested in uses on loop PHI backedges that are inductions. Otherwise we'll needlessly vectorize the IV increment - and cause hybrid SLP for SLP inductions. Unless the PHI is live - of course. */ + and cause hybrid SLP for SLP inductions. */ else if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_induction_def - && ! STMT_VINFO_LIVE_P (stmt_vinfo) && (PHI_ARG_DEF_FROM_EDGE (stmt_vinfo->stmt, loop_latch_edge (bb->loop_father)) - == use)) + == use) + && (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo) + || (gimple_bb (stmt_vinfo->stmt) + != LOOP_VINFO_LOOP (loop_vinfo)->header))) { if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -670,7 +655,6 @@ process_use (stmt_vec_info stmt_vinfo, tree use, loop_vec_info loop_vinfo, return opt_result::success (); } - vect_mark_relevant (worklist, dstmt_vinfo, relevant, false); return opt_result::success (); } @@ -714,22 +698,36 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo, bool *fatal) bb = bbs[i]; for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si)) { + if (virtual_operand_p (gimple_phi_result (gsi_stmt (si)))) + continue; stmt_vec_info phi_info = loop_vinfo->lookup_stmt (gsi_stmt (si)); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "init: phi relevant? %G", phi_info->stmt); if (vect_stmt_relevant_p (phi_info, loop_vinfo, &relevant, &live_p)) - vect_mark_relevant (&worklist, phi_info, relevant, live_p); + { + if (STMT_VINFO_DEF_TYPE (phi_info) == vect_unknown_def_type) + return opt_result::failure_at + (*si, "not vectorized: unhandled relevant PHI: %G", *si); + vect_mark_relevant (&worklist, phi_info, relevant, live_p); + } } - for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) + for (si = gsi_after_labels (bb); !gsi_end_p (si); gsi_next (&si)) { - if (is_gimple_debug (gsi_stmt (si))) + gimple *stmt = gsi_stmt (si); + if (is_gimple_debug (stmt)) continue; - stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si)); + stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, - "init: stmt relevant? %G", stmt_info->stmt); + "init: stmt relevant? %G", stmt); + + if (gimple_get_lhs (stmt) == NULL_TREE + && !is_a <gcond *> (stmt) + && !is_a <gcall *> (stmt)) + return opt_result::failure_at + (stmt, "not vectorized: irregular stmt: %G", stmt); if (vect_stmt_relevant_p (stmt_info, loop_vinfo, &relevant, &live_p)) vect_mark_relevant (&worklist, stmt_info, relevant, live_p); @@ -875,7 +873,9 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo, bool *fatal) if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo)) { gather_scatter_info gs_info; - if (!vect_check_gather_scatter (stmt_vinfo, loop_vinfo, &gs_info)) + if (!vect_check_gather_scatter (stmt_vinfo, + STMT_VINFO_VECTYPE (stmt_vinfo), + loop_vinfo, &gs_info)) gcc_unreachable (); opt_result res = process_use (stmt_vinfo, gs_info.offset, loop_vinfo, relevant, @@ -894,16 +894,11 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo, bool *fatal) /* Function vect_model_simple_cost. - Models cost for simple operations, i.e. those that only emit ncopies of a - single op. Right now, this does not account for multiple insns that could - be generated for the single vector op. We will handle that shortly. */ + Models cost for simple operations, i.e. those that only emit N operations + of the same KIND. */ static void -vect_model_simple_cost (vec_info *, - stmt_vec_info stmt_info, int ncopies, - enum vect_def_type *dt, - int ndts, - slp_tree node, +vect_model_simple_cost (vec_info *vinfo, int n, slp_tree node, stmt_vector_for_cost *cost_vec, vect_cost_for_stmt kind = vector_stmt) { @@ -911,22 +906,10 @@ vect_model_simple_cost (vec_info *, gcc_assert (cost_vec != NULL); - /* ??? Somehow we need to fix this at the callers. */ - if (node) - ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (node); - - if (!node) - /* Cost the "broadcast" of a scalar operand in to a vector operand. - Use scalar_to_vec to cost the broadcast, as elsewhere in the vector - cost model. */ - for (int i = 0; i < ndts; i++) - if (dt[i] == vect_constant_def || dt[i] == vect_external_def) - prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec, - stmt_info, 0, vect_prologue); + n *= vect_get_num_copies (vinfo, node); /* Pass the inside-of-loop statements to the target-specific cost model. */ - inside_cost += record_stmt_cost (cost_vec, ncopies, kind, - stmt_info, 0, vect_body); + inside_cost += record_stmt_cost (cost_vec, n, kind, node, 0, vect_body); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -944,8 +927,7 @@ vect_model_simple_cost (vec_info *, is true the stmt is doing widening arithmetic. */ static void -vect_model_promotion_demotion_cost (stmt_vec_info stmt_info, - enum vect_def_type *dt, +vect_model_promotion_demotion_cost (slp_tree slp_node, unsigned int ncopies, int pwr, stmt_vector_for_cost *cost_vec, bool widen_arith) @@ -958,16 +940,10 @@ vect_model_promotion_demotion_cost (stmt_vec_info stmt_info, inside_cost += record_stmt_cost (cost_vec, ncopies, widen_arith ? vector_stmt : vec_promote_demote, - stmt_info, 0, vect_body); + slp_node, 0, vect_body); ncopies *= 2; } - /* FORNOW: Assuming maximum 2 args per stmts. */ - for (i = 0; i < 2; i++) - if (dt[i] == vect_constant_def || dt[i] == vect_external_def) - prologue_cost += record_stmt_cost (cost_vec, 1, vector_stmt, - stmt_info, 0, vect_prologue); - if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "vect_model_promotion_demotion_cost: inside_cost = %d, " @@ -1013,13 +989,15 @@ vect_get_store_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node, unsigned int *inside_cost, stmt_vector_for_cost *body_cost_vec) { + tree vectype + = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info); switch (alignment_support_scheme) { case dr_aligned: { *inside_cost += record_stmt_cost (body_cost_vec, ncopies, - vector_store, stmt_info, slp_node, 0, - vect_body); + vector_store, stmt_info, slp_node, + vectype, 0, vect_body); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -1032,7 +1010,7 @@ vect_get_store_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node, /* Here, we assign an additional cost for the unaligned store. */ *inside_cost += record_stmt_cost (body_cost_vec, ncopies, unaligned_store, stmt_info, slp_node, - misalignment, vect_body); + vectype, misalignment, vect_body); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "vect_model_store_cost: unaligned supported by " @@ -1066,12 +1044,15 @@ vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node, stmt_vector_for_cost *body_cost_vec, bool record_prologue_costs) { + tree vectype + = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info); switch (alignment_support_scheme) { case dr_aligned: { *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load, - stmt_info, slp_node, 0, vect_body); + stmt_info, slp_node, vectype, + 0, vect_body); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -1084,7 +1065,7 @@ vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node, /* Here, we assign an additional cost for the unaligned load. */ *inside_cost += record_stmt_cost (body_cost_vec, ncopies, unaligned_load, stmt_info, slp_node, - misalignment, vect_body); + vectype, misalignment, vect_body); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -1096,18 +1077,19 @@ vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node, case dr_explicit_realign: { *inside_cost += record_stmt_cost (body_cost_vec, ncopies * 2, - vector_load, stmt_info, slp_node, 0, - vect_body); + vector_load, stmt_info, slp_node, + vectype, 0, vect_body); *inside_cost += record_stmt_cost (body_cost_vec, ncopies, - vec_perm, stmt_info, slp_node, 0, - vect_body); + vec_perm, stmt_info, slp_node, + vectype, 0, vect_body); /* FIXME: If the misalignment remains fixed across the iterations of the containing loop, the following cost should be added to the prologue costs. */ if (targetm.vectorize.builtin_mask_for_load) *inside_cost += record_stmt_cost (body_cost_vec, 1, vector_stmt, - stmt_info, slp_node, 0, vect_body); + stmt_info, slp_node, vectype, + 0, vect_body); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -1133,17 +1115,21 @@ vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node, { *prologue_cost += record_stmt_cost (prologue_cost_vec, 2, vector_stmt, stmt_info, - slp_node, 0, vect_prologue); + slp_node, vectype, + 0, vect_prologue); if (targetm.vectorize.builtin_mask_for_load) *prologue_cost += record_stmt_cost (prologue_cost_vec, 1, vector_stmt, stmt_info, - slp_node, 0, vect_prologue); + slp_node, vectype, + 0, vect_prologue); } *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load, - stmt_info, slp_node, 0, vect_body); + stmt_info, slp_node, vectype, + 0, vect_body); *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_perm, - stmt_info, slp_node, 0, vect_body); + stmt_info, slp_node, vectype, + 0, vect_body); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -1255,130 +1241,23 @@ vect_init_vector (vec_info *vinfo, stmt_vec_info stmt_info, tree val, tree type, } -/* Function vect_get_vec_defs_for_operand. - - OP is an operand in STMT_VINFO. This function returns a vector of - NCOPIES defs that will be used in the vectorized stmts for STMT_VINFO. - - In the case that OP is an SSA_NAME which is defined in the loop, then - STMT_VINFO_VEC_STMTS of the defining stmt holds the relevant defs. - - In case OP is an invariant or constant, a new stmt that creates a vector def - needs to be introduced. VECTYPE may be used to specify a required type for - vector invariant. */ - -void -vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info stmt_vinfo, - unsigned ncopies, - tree op, vec<tree> *vec_oprnds, tree vectype) -{ - gimple *def_stmt; - enum vect_def_type dt; - bool is_simple_use; - loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); - - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "vect_get_vec_defs_for_operand: %T\n", op); - - stmt_vec_info def_stmt_info; - is_simple_use = vect_is_simple_use (op, loop_vinfo, &dt, - &def_stmt_info, &def_stmt); - gcc_assert (is_simple_use); - if (def_stmt && dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, " def_stmt = %G", def_stmt); - - vec_oprnds->create (ncopies); - if (dt == vect_constant_def || dt == vect_external_def) - { - tree stmt_vectype = STMT_VINFO_VECTYPE (stmt_vinfo); - tree vector_type; - - if (vectype) - vector_type = vectype; - else if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op)) - && VECTOR_BOOLEAN_TYPE_P (stmt_vectype)) - vector_type = truth_type_for (stmt_vectype); - else - vector_type = get_vectype_for_scalar_type (loop_vinfo, TREE_TYPE (op)); - - gcc_assert (vector_type); - /* A masked load can have a default SSA definition as else operand. - We should "vectorize" this instead of creating a duplicate from the - scalar default. */ - tree vop; - if (TREE_CODE (op) == SSA_NAME - && SSA_NAME_IS_DEFAULT_DEF (op) - && VAR_P (SSA_NAME_VAR (op))) - vop = get_or_create_ssa_default_def (cfun, - create_tmp_var (vector_type)); - else - vop = vect_init_vector (vinfo, stmt_vinfo, op, vector_type, NULL); - while (ncopies--) - vec_oprnds->quick_push (vop); - } - else - { - def_stmt_info = vect_stmt_to_vectorize (def_stmt_info); - gcc_assert (STMT_VINFO_VEC_STMTS (def_stmt_info).length () == ncopies); - for (unsigned i = 0; i < ncopies; ++i) - vec_oprnds->quick_push (gimple_get_lhs - (STMT_VINFO_VEC_STMTS (def_stmt_info)[i])); - } -} - - /* Get vectorized definitions for OP0 and OP1. */ void -vect_get_vec_defs (vec_info *vinfo, stmt_vec_info stmt_info, slp_tree slp_node, - unsigned ncopies, - tree op0, tree vectype0, vec<tree> *vec_oprnds0, - tree op1, tree vectype1, vec<tree> *vec_oprnds1, - tree op2, tree vectype2, vec<tree> *vec_oprnds2, - tree op3, tree vectype3, vec<tree> *vec_oprnds3) -{ - if (slp_node) - { - if (op0) - vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0); - if (op1) - vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1); - if (op2) - vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2); - if (op3) - vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3); - } - else - { - if (op0) - vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies, - op0, vec_oprnds0, vectype0); - if (op1) - vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies, - op1, vec_oprnds1, vectype1); - if (op2) - vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies, - op2, vec_oprnds2, vectype2); - if (op3) - vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies, - op3, vec_oprnds3, vectype3); - } -} - -void -vect_get_vec_defs (vec_info *vinfo, stmt_vec_info stmt_info, slp_tree slp_node, - unsigned ncopies, +vect_get_vec_defs (vec_info *, slp_tree slp_node, tree op0, vec<tree> *vec_oprnds0, tree op1, vec<tree> *vec_oprnds1, tree op2, vec<tree> *vec_oprnds2, tree op3, vec<tree> *vec_oprnds3) { - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, - op0, NULL_TREE, vec_oprnds0, - op1, NULL_TREE, vec_oprnds1, - op2, NULL_TREE, vec_oprnds2, - op3, NULL_TREE, vec_oprnds3); + if (op0) + vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0); + if (op1) + vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1); + if (op2) + vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2); + if (op3) + vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3); } /* Helper function called by vect_finish_replace_stmt and @@ -1535,25 +1414,41 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype, slp_tree slp_node, vec_load_store_type vls_type, int group_size, - vect_memory_access_type - memory_access_type, - gather_scatter_info *gs_info, - tree scalar_mask, + vect_load_store_data *ls, + slp_tree mask_node, vec<int> *elsvals = nullptr) { + vect_memory_access_type memory_access_type = ls->memory_access_type; + /* Invariant loads need no special support. */ if (memory_access_type == VMAT_INVARIANT) return; - unsigned int nvectors = vect_get_num_copies (loop_vinfo, slp_node, vectype); + /* Figure whether the mask is uniform. scalar_mask is used to + populate the scalar_cond_masked_set. */ + tree scalar_mask = NULL_TREE; + if (mask_node) + for (unsigned i = 0; i < SLP_TREE_LANES (mask_node); ++i) + { + tree def = vect_get_slp_scalar_def (mask_node, i); + if (!def + || (scalar_mask && def != scalar_mask)) + { + scalar_mask = NULL; + break; + } + else + scalar_mask = def; + } + + unsigned int nvectors = vect_get_num_copies (loop_vinfo, slp_node); vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo); vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo); machine_mode vecmode = TYPE_MODE (vectype); bool is_load = (vls_type == VLS_LOAD); if (memory_access_type == VMAT_LOAD_STORE_LANES) { - if (slp_node) - nvectors /= group_size; + nvectors /= group_size; internal_fn ifn = (is_load ? vect_load_lanes_supported (vectype, group_size, true, elsvals) @@ -1575,7 +1470,7 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype, return; } - if (memory_access_type == VMAT_GATHER_SCATTER) + if (mat_gather_scatter_p (memory_access_type)) { internal_fn ifn = (is_load ? IFN_MASK_GATHER_LOAD @@ -1583,17 +1478,33 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype, internal_fn len_ifn = (is_load ? IFN_MASK_LEN_GATHER_LOAD : IFN_MASK_LEN_SCATTER_STORE); + stmt_vec_info repr = SLP_TREE_REPRESENTATIVE (slp_node); + tree off_vectype = (STMT_VINFO_GATHER_SCATTER_P (repr) + ? SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0]) + : ls->strided_offset_vectype); + tree memory_type = TREE_TYPE (DR_REF (STMT_VINFO_DR_INFO (repr)->dr)); + int scale = SLP_TREE_GS_SCALE (slp_node); + + /* The following "supported" checks just verify what we established in + get_load_store_type and don't try different offset types. + Therefore, off_vectype must be a supported offset type. In case + we chose a different one use this instead. */ + if (ls->supported_offset_vectype) + off_vectype = ls->supported_offset_vectype; + /* Same for scale. */ + if (ls->supported_scale) + scale = ls->supported_scale; + if (internal_gather_scatter_fn_supported_p (len_ifn, vectype, - gs_info->memory_type, - gs_info->offset_vectype, - gs_info->scale, + memory_type, + off_vectype, scale, elsvals)) vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1); else if (internal_gather_scatter_fn_supported_p (ifn, vectype, - gs_info->memory_type, - gs_info->offset_vectype, - gs_info->scale, - elsvals)) + memory_type, + off_vectype, scale, + elsvals) + || memory_access_type == VMAT_GATHER_SCATTER_LEGACY) vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype, scalar_mask); else @@ -1608,8 +1519,7 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype, return; } - if (memory_access_type != VMAT_CONTIGUOUS - && memory_access_type != VMAT_CONTIGUOUS_PERMUTE) + if (memory_access_type != VMAT_CONTIGUOUS) { /* Element X of the data must come from iteration i * VF + X of the scalar loop. We need more work to support other mappings. */ @@ -1632,7 +1542,7 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype, } /* We might load more scalars than we need for permuting SLP loads. - We checked in get_group_load_store_type that the extra elements + We checked in get_load_store_type that the extra elements don't leak into a new vector. */ auto group_memory_nvectors = [](poly_uint64 size, poly_uint64 nunits) { @@ -1720,7 +1630,7 @@ prepare_vec_mask (loop_vec_info loop_vinfo, tree mask_type, tree loop_mask, else values will be stored in the vector ELSVALS points to. */ static bool -vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, +vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, tree vectype, loop_vec_info loop_vinfo, bool masked_p, gather_scatter_info *gs_info, vec<int> *elsvals) @@ -1738,7 +1648,6 @@ vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, } /* Get the number of bits in an element. */ - tree vectype = STMT_VINFO_VECTYPE (stmt_info); scalar_mode element_mode = SCALAR_TYPE_MODE (TREE_TYPE (vectype)); unsigned int element_bits = GET_MODE_BITSIZE (element_mode); @@ -1778,10 +1687,13 @@ vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, /* See whether the target supports the operation with an offset no narrower than OFFSET_TYPE. */ tree memory_type = TREE_TYPE (DR_REF (dr)); + tree tmp_offset_vectype; + int tmp_scale; if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p, - vectype, memory_type, offset_type, scale, + vectype, memory_type, offset_type, + scale, &tmp_scale, &gs_info->ifn, &gs_info->offset_vectype, - elsvals) + &tmp_offset_vectype, elsvals) || gs_info->ifn == IFN_LAST) continue; @@ -1789,9 +1701,11 @@ vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET, but we don't need to store that here. */ gs_info->base = NULL_TREE; + gs_info->alias_ptr = build_int_cst + (reference_alias_ptr_type (DR_REF (dr)), + get_object_alignment (DR_REF (dr))); gs_info->element_type = TREE_TYPE (vectype); gs_info->offset = fold_convert (offset_type, step); - gs_info->offset_dt = vect_constant_def; gs_info->scale = scale; gs_info->memory_type = memory_type; return true; @@ -1805,6 +1719,97 @@ vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, return false; } +/* Return true if we can use gather/scatter or strided internal functions + to vectorize STMT_INFO, which is a grouped or strided load or store + with multiple lanes and will be implemented by a type-punned access + of a vector with element size that matches the number of lanes. + + MASKED_P is true if load or store is conditional. + When returning true, fill in GS_INFO with the information required to + perform the operation. Also, store the punning type in PUNNED_VECTYPE. + + If successful and ELSVALS is nonzero the supported + else values will be stored in the vector ELSVALS points to. */ + +static bool +vect_use_grouped_gather (dr_vec_info *dr_info, tree vectype, + loop_vec_info loop_vinfo, bool masked_p, + unsigned int nelts, + gather_scatter_info *info, vec<int> *elsvals, + tree *pun_vectype) +{ + data_reference *dr = dr_info->dr; + + /* TODO: We can support nelts > BITS_PER_UNIT or non-power-of-two by + multiple gathers/scatter. */ + if (nelts > BITS_PER_UNIT || !pow2p_hwi (nelts)) + return false; + + /* Pun the vectype with one of the same size but an element spanning + NELTS elements of VECTYPE. + The punned type of a V16QI with NELTS = 4 would be V4SI. + */ + tree tmp; + unsigned int pieces; + if (!can_div_trunc_p (TYPE_VECTOR_SUBPARTS (vectype), nelts, &pieces) + || !pieces) + return false; + + *pun_vectype = vector_vector_composition_type (vectype, pieces, &tmp, true); + + if (!*pun_vectype || !VECTOR_TYPE_P (*pun_vectype)) + return false; + + internal_fn ifn; + tree offset_vectype = *pun_vectype; + + internal_fn strided_ifn = DR_IS_READ (dr) + ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE; + + /* Check if we have a gather/scatter with the new type. We're just trying + with the type itself as offset for now. If not, check if we have a + strided load/store. These have fewer constraints (for example no offset + type must exist) so it is possible that even though a gather/scatter is + not available we still have a strided load/store. */ + bool ok = false; + tree tmp_vectype; + int tmp_scale; + if (vect_gather_scatter_fn_p + (loop_vinfo, DR_IS_READ (dr), masked_p, *pun_vectype, + TREE_TYPE (*pun_vectype), *pun_vectype, 1, &tmp_scale, &ifn, + &offset_vectype, &tmp_vectype, elsvals)) + ok = true; + else if (internal_strided_fn_supported_p (strided_ifn, *pun_vectype, + elsvals)) + { + /* Use gather/scatter IFNs, vect_get_strided_load_store_ops + will switch back to the strided variants. */ + ifn = DR_IS_READ (dr) ? IFN_MASK_LEN_GATHER_LOAD : + IFN_MASK_LEN_SCATTER_STORE; + ok = true; + } + + if (ok) + { + info->ifn = ifn; + info->decl = NULL_TREE; + info->base = dr->ref; + info->alias_ptr = build_int_cst + (reference_alias_ptr_type (DR_REF (dr)), + get_object_alignment (DR_REF (dr))); + info->element_type = TREE_TYPE (*pun_vectype); + info->offset_vectype = offset_vectype; + /* No need to set the offset, vect_get_strided_load_store_ops + will do that. */ + info->scale = 1; + info->memory_type = TREE_TYPE (DR_REF (dr)); + return true; + } + + return false; +} + + /* Return true if we can use gather/scatter internal functions to vectorize STMT_INFO, which is a grouped or strided load or store. MASKED_P is true if load or store is conditional. When returning @@ -1815,22 +1820,27 @@ vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, else values will be stored in the vector ELSVALS points to. */ static bool -vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info, +vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info, tree vectype, loop_vec_info loop_vinfo, bool masked_p, gather_scatter_info *gs_info, - vec<int> *elsvals) + vec<int> *elsvals, + unsigned int group_size, + bool single_element_p) { - if (!vect_check_gather_scatter (stmt_info, loop_vinfo, gs_info, elsvals) + if (!vect_check_gather_scatter (stmt_info, vectype, + loop_vinfo, gs_info, elsvals) || gs_info->ifn == IFN_LAST) - return vect_truncate_gather_scatter_offset (stmt_info, loop_vinfo, - masked_p, gs_info, elsvals); - - tree old_offset_type = TREE_TYPE (gs_info->offset); - tree new_offset_type = TREE_TYPE (gs_info->offset_vectype); + { + if (!vect_truncate_gather_scatter_offset (stmt_info, vectype, loop_vinfo, + masked_p, gs_info, elsvals)) + return false; + } - gcc_assert (TYPE_PRECISION (new_offset_type) - >= TYPE_PRECISION (old_offset_type)); - gs_info->offset = fold_convert (new_offset_type, gs_info->offset); + if (!single_element_p + && !targetm.vectorize.prefer_gather_scatter (TYPE_MODE (vectype), + gs_info->scale, + group_size)) + return false; if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -1956,12 +1966,14 @@ vect_get_store_rhs (stmt_vec_info stmt_info) /* Function VECTOR_VECTOR_COMPOSITION_TYPE - This function returns a vector type which can be composed with NETLS pieces, + This function returns a vector type which can be composed with NELTS pieces, whose type is recorded in PTYPE. VTYPE should be a vector type, and has the same vector size as the return vector. It checks target whether supports pieces-size vector mode for construction firstly, if target fails to, check pieces-size scalar mode for construction further. It returns NULL_TREE if - fails to find the available composition. + fails to find the available composition. If the caller only wants scalar + pieces where PTYPE e.g. is a possible gather/scatter element type + SCALAR_PTYPE_ONLY must be true. For example, for (vtype=V16QI, nelts=4), we can probably get: - V16QI with PTYPE V4QI. @@ -1969,7 +1981,8 @@ vect_get_store_rhs (stmt_vec_info stmt_info) - NULL_TREE. */ static tree -vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype) +vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype, + bool scalar_ptype_only) { gcc_assert (VECTOR_TYPE_P (vtype)); gcc_assert (known_gt (nelts, 0U)); @@ -1995,7 +2008,8 @@ vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype) scalar_mode elmode = SCALAR_TYPE_MODE (TREE_TYPE (vtype)); poly_uint64 inelts = pbsize / GET_MODE_BITSIZE (elmode); machine_mode rmode; - if (related_vector_mode (vmode, elmode, inelts).exists (&rmode) + if (!scalar_ptype_only + && related_vector_mode (vmode, elmode, inelts).exists (&rmode) && (convert_optab_handler (vec_init_optab, vmode, rmode) != CODE_FOR_nothing)) { @@ -2006,50 +2020,98 @@ vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype) /* Otherwise check if exists an integer type of the same piece size and if vec_init optab supports construction from it directly. */ if (int_mode_for_size (pbsize, 0).exists (&elmode) - && related_vector_mode (vmode, elmode, nelts).exists (&rmode) - && (convert_optab_handler (vec_init_optab, rmode, elmode) - != CODE_FOR_nothing)) + && related_vector_mode (vmode, elmode, nelts).exists (&rmode)) { - *ptype = build_nonstandard_integer_type (pbsize, 1); - return build_vector_type (*ptype, nelts); + if (scalar_ptype_only + || convert_optab_handler (vec_init_optab, rmode, elmode) + != CODE_FOR_nothing) + { + *ptype = build_nonstandard_integer_type (pbsize, 1); + return build_vector_type (*ptype, nelts); + } } } return NULL_TREE; } -/* A subroutine of get_load_store_type, with a subset of the same - arguments. Handle the case where STMT_INFO is part of a grouped load - or store. +/* Check if the load permutation of NODE only refers to a consecutive + subset of the group indices where GROUP_SIZE is the size of the + dataref's group. We also assert that the length of the permutation + divides the group size and is a power of two. + Such load permutations can be elided in strided access schemes as + we can "jump over" the gap they leave. */ + +bool +has_consecutive_load_permutation (slp_tree node, unsigned group_size) +{ + load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node); + if (!perm.exists () + || perm.length () <= 1 + || !pow2p_hwi (perm.length ()) + || group_size % perm.length ()) + return false; - For stores, the statements in the group are all consecutive - and there is no gap at the end. For loads, the statements in the - group might not be consecutive; there can be gaps between statements - as well as at the end. + return vect_load_perm_consecutive_p (node); +} - If we can use gather/scatter and ELSVALS is nonzero the supported - else values will be stored in the vector ELSVALS points to. -*/ + +/* Analyze load or store SLP_NODE of type VLS_TYPE. Return true + if there is a memory access type that the vectorized form can use, + storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers + or scatters, fill in GS_INFO accordingly. In addition + *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if + the target does not support the alignment scheme. *MISALIGNMENT + is set according to the alignment of the access (including + DR_MISALIGNMENT_UNKNOWN when it is unknown). + + MASKED_P is true if the statement is conditional on a vectorized mask. + VECTYPE is the vector type that the vectorized statements will use. + + If ELSVALS is nonzero the supported else values will be stored in the + vector ELSVALS points to. */ static bool -get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, - tree vectype, slp_tree slp_node, - bool masked_p, vec_load_store_type vls_type, - vect_memory_access_type *memory_access_type, - poly_int64 *poffset, - dr_alignment_support *alignment_support_scheme, - int *misalignment, - gather_scatter_info *gs_info, - internal_fn *lanes_ifn, - vec<int> *elsvals) +get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, + tree vectype, slp_tree slp_node, + bool masked_p, vec_load_store_type vls_type, + vect_load_store_data *ls) { + vect_memory_access_type *memory_access_type = &ls->memory_access_type; + poly_int64 *poffset = &ls->poffset; + dr_alignment_support *alignment_support_scheme + = &ls->alignment_support_scheme; + int *misalignment = &ls->misalignment; + internal_fn *lanes_ifn = &ls->lanes_ifn; + vec<int> *elsvals = &ls->elsvals; + tree *ls_type = &ls->ls_type; + bool *slp_perm = &ls->slp_perm; + unsigned *n_perms = &ls->n_perms; + tree *supported_offset_vectype = &ls->supported_offset_vectype; + int *supported_scale = &ls->supported_scale; loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); + poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL; stmt_vec_info first_stmt_info; unsigned int group_size; unsigned HOST_WIDE_INT gap; bool single_element_p; poly_int64 neg_ldst_offset = 0; + + *misalignment = DR_MISALIGNMENT_UNKNOWN; + *poffset = 0; + *ls_type = NULL_TREE; + *slp_perm = false; + *n_perms = -1U; + ls->subchain_p = false; + + bool perm_ok = true; + poly_int64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1; + + if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) + perm_ok = vect_transform_slp_perm_load (vinfo, slp_node, vNULL, NULL, + vf, true, n_perms); + if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) { first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info); @@ -2066,7 +2128,6 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, single_element_p = true; } dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info); - poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); /* True if the vectorized statements would access beyond the last statement in the group. */ @@ -2083,301 +2144,297 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, known at compile time. */ gcc_assert (!STMT_VINFO_STRIDED_P (first_stmt_info) || gap == 0); - /* Stores can't yet have gaps. */ - gcc_assert (slp_node || vls_type == VLS_LOAD || gap == 0); + /* For SLP vectorization we directly vectorize a subchain + without permutation. */ + if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) + first_dr_info = STMT_VINFO_DR_INFO (SLP_TREE_SCALAR_STMTS (slp_node)[0]); - if (slp_node) + if (STMT_VINFO_STRIDED_P (first_stmt_info)) { - /* For SLP vectorization we directly vectorize a subchain - without permutation. */ - if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) - first_dr_info - = STMT_VINFO_DR_INFO (SLP_TREE_SCALAR_STMTS (slp_node)[0]); - if (STMT_VINFO_STRIDED_P (first_stmt_info)) - /* Try to use consecutive accesses of as many elements as possible, - separated by the stride, until we have a complete vector. - Fall back to scalar accesses if that isn't possible. */ - *memory_access_type = VMAT_STRIDED_SLP; - else + /* Try to use consecutive accesses of as many elements as possible, + separated by the stride, until we have a complete vector. + Fall back to scalar accesses if that isn't possible. */ + *memory_access_type = VMAT_STRIDED_SLP; + + /* If the load permutation is consecutive we can reduce the group to + the elements the permutation accesses. Then we release the + permutation. */ + if (has_consecutive_load_permutation (slp_node, group_size)) + { + ls->subchain_p = true; + group_size = SLP_TREE_LANES (slp_node); + SLP_TREE_LOAD_PERMUTATION (slp_node).release (); + } + } + else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) + { + slp_tree offset_node = SLP_TREE_CHILDREN (slp_node)[0]; + tree offset_vectype = SLP_TREE_VECTYPE (offset_node); + int scale = SLP_TREE_GS_SCALE (slp_node); + tree memory_type = TREE_TYPE (DR_REF (first_dr_info->dr)); + tree tem; + if (vect_gather_scatter_fn_p (loop_vinfo, vls_type == VLS_LOAD, + masked_p, vectype, memory_type, + offset_vectype, scale, supported_scale, + &ls->gs.ifn, &tem, + supported_offset_vectype, elsvals)) { - int cmp = compare_step_with_zero (vinfo, stmt_info); - if (cmp < 0) + if (dump_enabled_p ()) { - if (single_element_p) - /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is - only correct for single element "interleaving" SLP. */ - *memory_access_type = get_negative_load_store_type - (vinfo, stmt_info, vectype, vls_type, 1, - &neg_ldst_offset); - else - { - /* Try to use consecutive accesses of DR_GROUP_SIZE elements, - separated by the stride, until we have a complete vector. - Fall back to scalar accesses if that isn't possible. */ - if (multiple_p (nunits, group_size)) - *memory_access_type = VMAT_STRIDED_SLP; - else - *memory_access_type = VMAT_ELEMENTWISE; - } - } - else if (cmp == 0 && loop_vinfo) + dump_printf_loc (MSG_NOTE, vect_location, + "gather/scatter with required " + "offset type " + "%T and offset scale %d.\n", + offset_vectype, scale); + if (*supported_offset_vectype) + dump_printf_loc (MSG_NOTE, vect_location, + " target supports offset type %T.\n", + *supported_offset_vectype); + if (*supported_scale) + dump_printf_loc (MSG_NOTE, vect_location, + " target supports offset scale %d.\n", + *supported_scale); + } + *memory_access_type = VMAT_GATHER_SCATTER_IFN; + } + else if (vls_type == VLS_LOAD + ? (targetm.vectorize.builtin_gather + && (ls->gs.decl + = targetm.vectorize.builtin_gather (vectype, + TREE_TYPE + (offset_vectype), + scale))) + : (targetm.vectorize.builtin_scatter + && (ls->gs.decl + = targetm.vectorize.builtin_scatter (vectype, + TREE_TYPE + (offset_vectype), + scale)))) + *memory_access_type = VMAT_GATHER_SCATTER_LEGACY; + else + { + /* GATHER_SCATTER_EMULATED_P. */ + if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant () + || !TYPE_VECTOR_SUBPARTS (offset_vectype).is_constant () + || VECTOR_BOOLEAN_TYPE_P (offset_vectype) + || !constant_multiple_p (TYPE_VECTOR_SUBPARTS (offset_vectype), + TYPE_VECTOR_SUBPARTS (vectype))) { - gcc_assert (vls_type == VLS_LOAD); - *memory_access_type = VMAT_INVARIANT; + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "unsupported vector types for emulated " + "gather.\n"); + return false; } - /* Try using LOAD/STORE_LANES. */ - else if (slp_node->ldst_lanes - && (*lanes_ifn - = (vls_type == VLS_LOAD - ? vect_load_lanes_supported (vectype, group_size, - masked_p, elsvals) - : vect_store_lanes_supported (vectype, group_size, - masked_p))) != IFN_LAST) - *memory_access_type = VMAT_LOAD_STORE_LANES; + *memory_access_type = VMAT_GATHER_SCATTER_EMULATED; + } + } + else + { + int cmp = compare_step_with_zero (vinfo, stmt_info); + if (cmp < 0) + { + if (single_element_p) + /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is + only correct for single element "interleaving" SLP. */ + *memory_access_type = get_negative_load_store_type + (vinfo, stmt_info, vectype, vls_type, 1, + &neg_ldst_offset); else - *memory_access_type = VMAT_CONTIGUOUS; + /* We can fall back to VMAT_STRIDED_SLP since that does + not care whether the stride between the group instances + is positive or negative. */ + *memory_access_type = VMAT_STRIDED_SLP; + } + else if (cmp == 0 && loop_vinfo) + { + gcc_assert (vls_type == VLS_LOAD); + *memory_access_type = VMAT_INVARIANT; + } + /* Try using LOAD/STORE_LANES. */ + else if (slp_node->ldst_lanes + && (*lanes_ifn + = (vls_type == VLS_LOAD + ? vect_load_lanes_supported (vectype, group_size, + masked_p, elsvals) + : vect_store_lanes_supported (vectype, group_size, + masked_p))) != IFN_LAST) + *memory_access_type = VMAT_LOAD_STORE_LANES; + else if (!loop_vinfo && slp_node->avoid_stlf_fail) + { + *memory_access_type = VMAT_ELEMENTWISE; + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "using element-wise load to avoid disrupting " + "cross iteration store-to-load forwarding\n"); + } + else + *memory_access_type = VMAT_CONTIGUOUS; - /* If this is single-element interleaving with an element - distance that leaves unused vector loads around fall back - to elementwise access if possible - we otherwise least - create very sub-optimal code in that case (and - blow up memory, see PR65518). */ - if (loop_vinfo - && single_element_p - && (*memory_access_type == VMAT_CONTIGUOUS - || *memory_access_type == VMAT_CONTIGUOUS_REVERSE) - && maybe_gt (group_size, TYPE_VECTOR_SUBPARTS (vectype))) - { - if (SLP_TREE_LANES (slp_node) == 1) + /* If this is single-element interleaving with an element + distance that leaves unused vector loads around fall back + to elementwise access if possible - we otherwise least + create very sub-optimal code in that case (and + blow up memory, see PR65518). */ + if (loop_vinfo + && single_element_p + && (*memory_access_type == VMAT_CONTIGUOUS + || *memory_access_type == VMAT_CONTIGUOUS_REVERSE) + && maybe_gt (group_size, TYPE_VECTOR_SUBPARTS (vectype))) + { + *memory_access_type = VMAT_ELEMENTWISE; + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "single-element interleaving not supported " + "for not adjacent vector loads, using " + "elementwise access\n"); + } + + /* Also fall back to elementwise access in case we did not lower a + permutation and cannot code generate it. */ + if (loop_vinfo + && *memory_access_type != VMAT_ELEMENTWISE + && SLP_TREE_LOAD_PERMUTATION (slp_node).exists () + && !perm_ok) + { + *memory_access_type = VMAT_ELEMENTWISE; + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "permutation not supported, using elementwise " + "access\n"); + } + + overrun_p = (loop_vinfo && gap != 0 + && *memory_access_type != VMAT_ELEMENTWISE); + if (overrun_p && vls_type != VLS_LOAD) + { + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "Grouped store with gaps requires" + " non-consecutive accesses\n"); + return false; + } + + unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (first_dr_info); + poly_int64 off = 0; + if (*memory_access_type == VMAT_CONTIGUOUS_REVERSE) + off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size; + + /* An overrun is fine if the trailing elements are smaller + than the alignment boundary B. Every vector access will + be a multiple of B and so we are guaranteed to access a + non-gap element in the same B-sized block. */ + if (overrun_p + && gap < (vect_known_alignment_in_bytes (first_dr_info, + vectype, off) / dr_size)) + overrun_p = false; + + /* When we have a contiguous access across loop iterations + but the access in the loop doesn't cover the full vector + we can end up with no gap recorded but still excess + elements accessed, see PR103116. Make sure we peel for + gaps if necessary and sufficient and give up if not. + + If there is a combination of the access not covering the full + vector and a gap recorded then we may need to peel twice. */ + bool large_vector_overrun_p = false; + if (loop_vinfo + && (*memory_access_type == VMAT_CONTIGUOUS + || *memory_access_type == VMAT_CONTIGUOUS_REVERSE) + && SLP_TREE_LOAD_PERMUTATION (slp_node).exists () + && !multiple_p (group_size * LOOP_VINFO_VECT_FACTOR (loop_vinfo), + nunits)) + large_vector_overrun_p = overrun_p = true; + + /* If the gap splits the vector in half and the target + can do half-vector operations avoid the epilogue peeling + by simply loading half of the vector only. Usually + the construction with an upper zero half will be elided. */ + dr_alignment_support alss; + int misalign = dr_misalignment (first_dr_info, vectype, off); + tree half_vtype; + poly_uint64 remain; + unsigned HOST_WIDE_INT tem, num; + if (overrun_p + && !masked_p + && *memory_access_type != VMAT_LOAD_STORE_LANES + && (((alss = vect_supportable_dr_alignment (vinfo, first_dr_info, + vectype, misalign))) + == dr_aligned + || alss == dr_unaligned_supported) + && can_div_trunc_p (group_size + * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap, + nunits, &tem, &remain) + && (known_eq (remain, 0u) + || (known_ne (remain, 0u) + && constant_multiple_p (nunits, remain, &num) + && (vector_vector_composition_type (vectype, num, &half_vtype) + != NULL_TREE)))) + overrun_p = false; + + if (overrun_p && !can_overrun_p) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "Peeling for outer loop is not supported\n"); + return false; + } + + /* Peeling for gaps assumes that a single scalar iteration + is enough to make sure the last vector iteration doesn't + access excess elements. */ + if (overrun_p + && (!can_div_trunc_p (group_size + * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap, + nunits, &tem, &remain) + || maybe_lt (remain + group_size, nunits))) + { + /* But peeling a single scalar iteration is enough if + we can use the next power-of-two sized partial + access and that is sufficiently small to be covered + by the single scalar iteration. */ + unsigned HOST_WIDE_INT cnunits, cvf, cremain, cpart_size; + if (masked_p + || !nunits.is_constant (&cnunits) + || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&cvf) + || (((cremain = (group_size * cvf - gap) % cnunits), true) + && ((cpart_size = (1 << ceil_log2 (cremain))), true) + && (cremain + group_size < cpart_size + || (vector_vector_composition_type (vectype, + cnunits / cpart_size, + &half_vtype) + == NULL_TREE)))) + { + /* If all fails we can still resort to niter masking unless + the vectors used are too big, so enforce the use of + partial vectors. */ + if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) + && !large_vector_overrun_p) { - *memory_access_type = VMAT_ELEMENTWISE; if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "single-element interleaving not supported " - "for not adjacent vector loads, using " - "elementwise access\n"); + "peeling for gaps insufficient for " + "access unless using partial " + "vectors\n"); + LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true; } else { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "single-element interleaving not supported " - "for not adjacent vector loads\n"); + "peeling for gaps insufficient for " + "access\n"); return false; } } - - /* For single-element interleaving also fall back to elementwise - access in case we did not lower a permutation and cannot - code generate it. */ - auto_vec<tree> temv; - unsigned n_perms; - if (loop_vinfo - && single_element_p - && SLP_TREE_LANES (slp_node) == 1 - && (*memory_access_type == VMAT_CONTIGUOUS - || *memory_access_type == VMAT_CONTIGUOUS_REVERSE) - && SLP_TREE_LOAD_PERMUTATION (slp_node).exists () - && !vect_transform_slp_perm_load - (loop_vinfo, slp_node, temv, NULL, - LOOP_VINFO_VECT_FACTOR (loop_vinfo), true, &n_perms)) - { - *memory_access_type = VMAT_ELEMENTWISE; - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "single-element interleaving permutation not " - "supported, using elementwise access\n"); - } - - overrun_p = (loop_vinfo && gap != 0 - && *memory_access_type != VMAT_ELEMENTWISE); - if (overrun_p && vls_type != VLS_LOAD) - { - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "Grouped store with gaps requires" - " non-consecutive accesses\n"); - return false; - } - - unsigned HOST_WIDE_INT dr_size - = vect_get_scalar_dr_size (first_dr_info); - poly_int64 off = 0; - if (*memory_access_type == VMAT_CONTIGUOUS_REVERSE) - off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size; - - /* An overrun is fine if the trailing elements are smaller - than the alignment boundary B. Every vector access will - be a multiple of B and so we are guaranteed to access a - non-gap element in the same B-sized block. */ - if (overrun_p - && gap < (vect_known_alignment_in_bytes (first_dr_info, - vectype, off) / dr_size)) - overrun_p = false; - - /* When we have a contiguous access across loop iterations - but the access in the loop doesn't cover the full vector - we can end up with no gap recorded but still excess - elements accessed, see PR103116. Make sure we peel for - gaps if necessary and sufficient and give up if not. - - If there is a combination of the access not covering the full - vector and a gap recorded then we may need to peel twice. */ - bool large_vector_overrun_p = false; - if (loop_vinfo - && (*memory_access_type == VMAT_CONTIGUOUS - || *memory_access_type == VMAT_CONTIGUOUS_REVERSE) - && SLP_TREE_LOAD_PERMUTATION (slp_node).exists () - && !multiple_p (group_size * LOOP_VINFO_VECT_FACTOR (loop_vinfo), - nunits)) - large_vector_overrun_p = overrun_p = true; - - /* If the gap splits the vector in half and the target - can do half-vector operations avoid the epilogue peeling - by simply loading half of the vector only. Usually - the construction with an upper zero half will be elided. */ - dr_alignment_support alss; - int misalign = dr_misalignment (first_dr_info, vectype, off); - tree half_vtype; - poly_uint64 remain; - unsigned HOST_WIDE_INT tem, num; - if (overrun_p - && !masked_p - && *memory_access_type != VMAT_LOAD_STORE_LANES - && (((alss = vect_supportable_dr_alignment (vinfo, first_dr_info, - vectype, misalign))) - == dr_aligned - || alss == dr_unaligned_supported) - && can_div_trunc_p (group_size - * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap, - nunits, &tem, &remain) - && (known_eq (remain, 0u) - || (known_ne (remain, 0u) - && constant_multiple_p (nunits, remain, &num) - && (vector_vector_composition_type (vectype, num, - &half_vtype) - != NULL_TREE)))) - overrun_p = false; - - if (overrun_p && !can_overrun_p) + else if (large_vector_overrun_p) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "Peeling for outer loop is not supported\n"); - return false; - } - - /* Peeling for gaps assumes that a single scalar iteration - is enough to make sure the last vector iteration doesn't - access excess elements. */ - if (overrun_p - && (!can_div_trunc_p (group_size - * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap, - nunits, &tem, &remain) - || maybe_lt (remain + group_size, nunits))) - { - /* But peeling a single scalar iteration is enough if - we can use the next power-of-two sized partial - access and that is sufficiently small to be covered - by the single scalar iteration. */ - unsigned HOST_WIDE_INT cnunits, cvf, cremain, cpart_size; - if (masked_p - || !nunits.is_constant (&cnunits) - || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&cvf) - || (((cremain = (group_size * cvf - gap) % cnunits), true) - && ((cpart_size = (1 << ceil_log2 (cremain))), true) - && (cremain + group_size < cpart_size - || vector_vector_composition_type - (vectype, cnunits / cpart_size, - &half_vtype) == NULL_TREE))) - { - /* If all fails we can still resort to niter masking unless - the vectors used are too big, so enforce the use of - partial vectors. */ - if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) - && !large_vector_overrun_p) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "peeling for gaps insufficient for " - "access unless using partial " - "vectors\n"); - LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true; - } - else - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "peeling for gaps insufficient for " - "access\n"); - return false; - } - } - else if (large_vector_overrun_p) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "can't operate on partial vectors because " - "only unmasked loads handle access " - "shortening required because of gaps at " - "the end of the access\n"); - LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false; - } - } - } - } - else - { - /* We can always handle this case using elementwise accesses, - but see if something more efficient is available. */ - *memory_access_type = VMAT_ELEMENTWISE; - - /* If there is a gap at the end of the group then these optimizations - would access excess elements in the last iteration. */ - bool would_overrun_p = (gap != 0); - /* An overrun is fine if the trailing elements are smaller than the - alignment boundary B. Every vector access will be a multiple of B - and so we are guaranteed to access a non-gap element in the - same B-sized block. */ - if (would_overrun_p - && !masked_p - && gap < (vect_known_alignment_in_bytes (first_dr_info, vectype) - / vect_get_scalar_dr_size (first_dr_info))) - would_overrun_p = false; - - if (!STMT_VINFO_STRIDED_P (first_stmt_info) - && (can_overrun_p || !would_overrun_p) - && compare_step_with_zero (vinfo, stmt_info) > 0) - { - /* First cope with the degenerate case of a single-element - vector. */ - if (known_eq (TYPE_VECTOR_SUBPARTS (vectype), 1U)) - ; - - else - { - /* Otherwise try using LOAD/STORE_LANES. */ - *lanes_ifn - = vls_type == VLS_LOAD - ? vect_load_lanes_supported (vectype, group_size, masked_p, - elsvals) - : vect_store_lanes_supported (vectype, group_size, - masked_p); - if (*lanes_ifn != IFN_LAST) - { - *memory_access_type = VMAT_LOAD_STORE_LANES; - overrun_p = would_overrun_p; - } - - /* If that fails, try using permuting loads. */ - else if (vls_type == VLS_LOAD - ? vect_grouped_load_supported (vectype, - single_element_p, - group_size) - : vect_grouped_store_supported (vectype, group_size)) - { - *memory_access_type = VMAT_CONTIGUOUS_PERMUTE; - overrun_p = would_overrun_p; - } + "can't operate on partial vectors because " + "only unmasked loads handle access " + "shortening required because of gaps at " + "the end of the access\n"); + LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false; } } } @@ -2389,21 +2446,62 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, on nearby locations. Or, even if it's a win over scalar code, it might not be a win over vectorizing at a lower VF, if that allows us to use contiguous accesses. */ - if ((*memory_access_type == VMAT_ELEMENTWISE - || *memory_access_type == VMAT_STRIDED_SLP) - && single_element_p - && (!slp_node || SLP_TREE_LANES (slp_node) == 1) - && loop_vinfo - && vect_use_strided_gather_scatters_p (stmt_info, loop_vinfo, - masked_p, gs_info, elsvals)) - *memory_access_type = VMAT_GATHER_SCATTER; + vect_memory_access_type grouped_gather_fallback = VMAT_UNINITIALIZED; + if (loop_vinfo + && (*memory_access_type == VMAT_ELEMENTWISE + || *memory_access_type == VMAT_STRIDED_SLP)) + { + gather_scatter_info gs_info; + if (SLP_TREE_LANES (slp_node) == 1 + && (!SLP_TREE_LOAD_PERMUTATION (slp_node).exists () + || single_element_p) + && vect_use_strided_gather_scatters_p (stmt_info, vectype, loop_vinfo, + masked_p, &gs_info, elsvals, + group_size, single_element_p)) + { + /* vect_use_strided_gather_scatters_p does not save the actually + supported scale and offset type so do that here. + We need it later in check_load_store_for_partial_vectors + where we only check if the given internal function is supported + (to choose whether to use the IFN, LEGACY, or EMULATED flavor + of gather/scatter) and don't re-do the full analysis. */ + tree tmp; + gcc_assert (vect_gather_scatter_fn_p + (loop_vinfo, vls_type == VLS_LOAD, masked_p, vectype, + gs_info.memory_type, TREE_TYPE (gs_info.offset), + gs_info.scale, supported_scale, &gs_info.ifn, + &tmp, supported_offset_vectype, elsvals)); + + SLP_TREE_GS_SCALE (slp_node) = gs_info.scale; + SLP_TREE_GS_BASE (slp_node) = error_mark_node; + ls->gs.ifn = gs_info.ifn; + ls->strided_offset_vectype = gs_info.offset_vectype; + *memory_access_type = VMAT_GATHER_SCATTER_IFN; + } + else if (SLP_TREE_LANES (slp_node) > 1 + && !masked_p + && !single_element_p + && vect_use_grouped_gather (STMT_VINFO_DR_INFO (stmt_info), + vectype, loop_vinfo, + masked_p, group_size, + &gs_info, elsvals, ls_type)) + { + SLP_TREE_GS_SCALE (slp_node) = gs_info.scale; + SLP_TREE_GS_BASE (slp_node) = error_mark_node; + grouped_gather_fallback = *memory_access_type; + *memory_access_type = VMAT_GATHER_SCATTER_IFN; + ls->gs.ifn = gs_info.ifn; + vectype = *ls_type; + ls->strided_offset_vectype = gs_info.offset_vectype; + } + } if (*memory_access_type == VMAT_CONTIGUOUS_DOWN || *memory_access_type == VMAT_CONTIGUOUS_REVERSE) *poffset = neg_ldst_offset; - if (*memory_access_type == VMAT_GATHER_SCATTER - || *memory_access_type == VMAT_ELEMENTWISE + if (*memory_access_type == VMAT_ELEMENTWISE + || *memory_access_type == VMAT_GATHER_SCATTER_LEGACY || *memory_access_type == VMAT_STRIDED_SLP || *memory_access_type == VMAT_INVARIANT) { @@ -2412,29 +2510,26 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, } else { - *misalignment = dr_misalignment (first_dr_info, vectype, *poffset); + if (mat_gather_scatter_p (*memory_access_type) + && !first_dr_info) + *misalignment = DR_MISALIGNMENT_UNKNOWN; + else + *misalignment = dr_misalignment (first_dr_info, vectype, *poffset); *alignment_support_scheme - = vect_supportable_dr_alignment (vinfo, first_dr_info, vectype, - *misalignment); - } - - if (vls_type != VLS_LOAD && first_stmt_info == stmt_info) - { - /* STMT is the leader of the group. Check the operands of all the - stmts of the group. */ - stmt_vec_info next_stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info); - while (next_stmt_info) - { - tree op = vect_get_store_rhs (next_stmt_info); - enum vect_def_type dt; - if (!vect_is_simple_use (op, vinfo, &dt)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "use not simple.\n"); - return false; - } - next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info); + = vect_supportable_dr_alignment + (vinfo, first_dr_info, vectype, *misalignment, + mat_gather_scatter_p (*memory_access_type)); + if (grouped_gather_fallback != VMAT_UNINITIALIZED + && *alignment_support_scheme != dr_aligned + && *alignment_support_scheme != dr_unaligned_supported) + { + /* No supportable alignment for a grouped gather, fall back to the + original memory access type. Even though VMAT_STRIDED_SLP might + also try aligned vector loads it can still choose vector + construction from scalars. */ + *memory_access_type = grouped_gather_fallback; + *alignment_support_scheme = dr_unaligned_supported; + *misalignment = DR_MISALIGNMENT_UNKNOWN; } } @@ -2448,145 +2543,6 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true; } - return true; -} - -/* Analyze load or store statement STMT_INFO of type VLS_TYPE. Return true - if there is a memory access type that the vectorized form can use, - storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers - or scatters, fill in GS_INFO accordingly. In addition - *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if - the target does not support the alignment scheme. *MISALIGNMENT - is set according to the alignment of the access (including - DR_MISALIGNMENT_UNKNOWN when it is unknown). - - SLP says whether we're performing SLP rather than loop vectorization. - MASKED_P is true if the statement is conditional on a vectorized mask. - VECTYPE is the vector type that the vectorized statements will use. - NCOPIES is the number of vector statements that will be needed. - - If ELSVALS is nonzero the supported else values will be stored in the - vector ELSVALS points to. */ - -static bool -get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, - tree vectype, slp_tree slp_node, - bool masked_p, vec_load_store_type vls_type, - unsigned int ncopies, - vect_memory_access_type *memory_access_type, - poly_int64 *poffset, - dr_alignment_support *alignment_support_scheme, - int *misalignment, - gather_scatter_info *gs_info, - internal_fn *lanes_ifn, - vec<int> *elsvals = nullptr) -{ - loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); - poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); - *misalignment = DR_MISALIGNMENT_UNKNOWN; - *poffset = 0; - if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) - { - *memory_access_type = VMAT_GATHER_SCATTER; - if (!vect_check_gather_scatter (stmt_info, loop_vinfo, gs_info, - elsvals)) - gcc_unreachable (); - /* When using internal functions, we rely on pattern recognition - to convert the type of the offset to the type that the target - requires, with the result being a call to an internal function. - If that failed for some reason (e.g. because another pattern - took priority), just handle cases in which the offset already - has the right type. */ - else if (gs_info->ifn != IFN_LAST - && !is_gimple_call (stmt_info->stmt) - && !tree_nop_conversion_p (TREE_TYPE (gs_info->offset), - TREE_TYPE (gs_info->offset_vectype))) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "%s offset requires a conversion\n", - vls_type == VLS_LOAD ? "gather" : "scatter"); - return false; - } - else if (!vect_is_simple_use (gs_info->offset, vinfo, - &gs_info->offset_dt, - &gs_info->offset_vectype)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "%s index use not simple.\n", - vls_type == VLS_LOAD ? "gather" : "scatter"); - return false; - } - else if (gs_info->ifn == IFN_LAST && !gs_info->decl) - { - if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant () - || !TYPE_VECTOR_SUBPARTS (gs_info->offset_vectype).is_constant () - || VECTOR_BOOLEAN_TYPE_P (gs_info->offset_vectype) - || !constant_multiple_p (TYPE_VECTOR_SUBPARTS - (gs_info->offset_vectype), - TYPE_VECTOR_SUBPARTS (vectype))) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "unsupported vector types for emulated " - "gather.\n"); - return false; - } - } - /* Gather-scatter accesses perform only component accesses, alignment - is irrelevant for them. */ - *alignment_support_scheme = dr_unaligned_supported; - } - else if (STMT_VINFO_GROUPED_ACCESS (stmt_info) || slp_node) - { - if (!get_group_load_store_type (vinfo, stmt_info, vectype, slp_node, - masked_p, - vls_type, memory_access_type, poffset, - alignment_support_scheme, - misalignment, gs_info, lanes_ifn, - elsvals)) - return false; - } - else if (STMT_VINFO_STRIDED_P (stmt_info)) - { - gcc_assert (!slp_node); - if (loop_vinfo - && vect_use_strided_gather_scatters_p (stmt_info, loop_vinfo, - masked_p, gs_info, elsvals)) - *memory_access_type = VMAT_GATHER_SCATTER; - else - *memory_access_type = VMAT_ELEMENTWISE; - /* Alignment is irrelevant here. */ - *alignment_support_scheme = dr_unaligned_supported; - } - else - { - int cmp = compare_step_with_zero (vinfo, stmt_info); - if (cmp == 0) - { - gcc_assert (vls_type == VLS_LOAD); - *memory_access_type = VMAT_INVARIANT; - /* Invariant accesses perform only component accesses, alignment - is irrelevant for them. */ - *alignment_support_scheme = dr_unaligned_supported; - } - else - { - if (cmp < 0) - *memory_access_type = get_negative_load_store_type - (vinfo, stmt_info, vectype, vls_type, ncopies, poffset); - else - *memory_access_type = VMAT_CONTIGUOUS; - *misalignment = dr_misalignment (STMT_VINFO_DR_INFO (stmt_info), - vectype, *poffset); - *alignment_support_scheme - = vect_supportable_dr_alignment (vinfo, - STMT_VINFO_DR_INFO (stmt_info), - vectype, *misalignment); - } - } - if ((*memory_access_type == VMAT_ELEMENTWISE || *memory_access_type == VMAT_STRIDED_SLP) && !nunits.is_constant ()) @@ -2598,7 +2554,6 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, return false; } - /* Checks if all scalar iterations are known to be inbounds. */ bool inbounds = DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (stmt_info)); @@ -2609,7 +2564,7 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, if (loop_vinfo && dr_safe_speculative_read_required (stmt_info) && LOOP_VINFO_EARLY_BREAKS (loop_vinfo) - && (*memory_access_type == VMAT_GATHER_SCATTER + && (mat_gather_scatter_p (*memory_access_type) || *memory_access_type == VMAT_STRIDED_SLP)) { if (dump_enabled_p ()) @@ -2618,85 +2573,42 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, "alignment. With non-contiguous memory vectorization" " could read out of bounds at %G ", STMT_VINFO_STMT (stmt_info)); - if (inbounds) - LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true; - else - return false; + if (inbounds) + LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true; + else + return false; } /* If this DR needs alignment for correctness, we must ensure the target alignment is a constant power-of-two multiple of the amount read per vector iteration or force masking. */ if (dr_safe_speculative_read_required (stmt_info) - && *alignment_support_scheme == dr_aligned) + && (*alignment_support_scheme == dr_aligned + && !mat_gather_scatter_p (*memory_access_type))) { /* We can only peel for loops, of course. */ gcc_checking_assert (loop_vinfo); + poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); + poly_uint64 read_amount + = vf * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))); + if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) + read_amount *= group_size; + auto target_alignment = DR_TARGET_ALIGNMENT (STMT_VINFO_DR_INFO (stmt_info)); - unsigned HOST_WIDE_INT target_align; - - bool group_aligned = false; - if (target_alignment.is_constant (&target_align) - && nunits.is_constant ()) - { - poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); - auto vectype_size - = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))); - poly_uint64 required_alignment = vf * vectype_size; - /* If we have a grouped access we require that the alignment be N * elem. */ - if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) - required_alignment *= - DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info)); - if (!multiple_p (target_alignment, required_alignment)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "desired alignment %wu not met. Instead got %wu " - "for DR alignment at %G", - required_alignment.to_constant (), - target_align, STMT_VINFO_STMT (stmt_info)); - return false; - } - - if (!pow2p_hwi (target_align)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "non-power-of-two vector alignment %wd " - "for DR alignment at %G", - target_align, STMT_VINFO_STMT (stmt_info)); - return false; - } - - /* For VLA we have to insert a runtime check that the vector loads - per iterations don't exceed a page size. For now we can use - POLY_VALUE_MAX as a proxy as we can't peel for VLA. */ - if (known_gt (required_alignment, (unsigned)param_min_pagesize)) + if (!multiple_p (target_alignment, read_amount)) + { + if (dump_enabled_p ()) { - if (dump_enabled_p ()) - { - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "alignment required for correctness ("); - dump_dec (MSG_MISSED_OPTIMIZATION, required_alignment); - dump_printf (MSG_NOTE, ") may exceed page size\n"); - } - return false; + dump_printf_loc (MSG_NOTE, vect_location, + "desired alignment not met, target was "); + dump_dec (MSG_NOTE, target_alignment); + dump_printf (MSG_NOTE, " previously, but read amount is "); + dump_dec (MSG_NOTE, read_amount); + dump_printf (MSG_NOTE, " at %G.\n", STMT_VINFO_STMT (stmt_info)); } - - group_aligned = true; - } - - /* There are multiple loads that have a misalignment that we couldn't - align. We would need LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P to - vectorize. */ - if (!group_aligned) - { - if (inbounds) - LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true; - else - return false; + return false; } /* When using a group access the first element may be aligned but the @@ -2709,7 +2621,7 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, such only the first load in the group is aligned, the rest are not. Because of this the permutes may break the alignment requirements that have been set, and as such we should for now, reject them. */ - if (slp_node && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) + if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -2718,6 +2630,33 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, STMT_VINFO_STMT (stmt_info)); return false; } + + /* Reject vectorization if we know the read mount per vector iteration + exceeds the min page size. */ + if (known_gt (read_amount, (unsigned) param_min_pagesize)) + { + if (dump_enabled_p ()) + { + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "alignment required for correctness ("); + dump_dec (MSG_MISSED_OPTIMIZATION, read_amount); + dump_printf (MSG_NOTE, ") may exceed page size.\n"); + } + return false; + } + + if (!vf.is_constant ()) + { + /* For VLA modes, we need a runtime check to ensure any speculative + read amount does not exceed the page size. Here we record the max + possible read amount for the check. */ + if (maybe_gt (read_amount, + LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo))) + LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo) = read_amount; + + /* For VLA modes, we must use partial vectors. */ + LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true; + } } if (*alignment_support_scheme == dr_unaligned_unsupported) @@ -2731,40 +2670,96 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, /* FIXME: At the moment the cost model seems to underestimate the cost of using elementwise accesses. This check preserves the traditional behavior until that can be fixed. */ - stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info); - if (!first_stmt_info) - first_stmt_info = stmt_info; if (*memory_access_type == VMAT_ELEMENTWISE && !STMT_VINFO_STRIDED_P (first_stmt_info) - && !(stmt_info == DR_GROUP_FIRST_ELEMENT (stmt_info) - && !DR_GROUP_NEXT_ELEMENT (stmt_info) - && !pow2p_hwi (DR_GROUP_SIZE (stmt_info)))) + && !(STMT_VINFO_GROUPED_ACCESS (stmt_info) + && single_element_p + && !pow2p_hwi (group_size))) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, "not falling back to elementwise accesses\n"); return false; } + + /* For BB vectorization build up the vector from existing scalar defs. */ + if (!loop_vinfo && *memory_access_type == VMAT_ELEMENTWISE) + return false; + + /* Some loads need to explicitly permute the loaded data if there + is a load permutation. Among those are: + - VMAT_ELEMENTWISE. + - VMAT_STRIDED_SLP. + - VMAT_GATHER_SCATTER: + - Strided gather (fallback for VMAT_STRIDED_SLP if #lanes == 1). + - Grouped strided gather (ditto but for #lanes > 1). + + For VMAT_ELEMENTWISE we can fold the load permutation into the + individual indices we access directly, eliding the permutation. + Strided gather only allows load permutations for the + single-element case. */ + + if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists () + && !(*memory_access_type == VMAT_ELEMENTWISE + || (mat_gather_scatter_p (*memory_access_type) + && SLP_TREE_LANES (slp_node) == 1 + && single_element_p))) + { + if (!loop_vinfo) + { + /* In BB vectorization we may not actually use a loaded vector + accessing elements in excess of DR_GROUP_SIZE. */ + stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (slp_node)[0]; + group_info = DR_GROUP_FIRST_ELEMENT (group_info); + unsigned HOST_WIDE_INT nunits; + unsigned j, k, maxk = 0; + FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (slp_node), j, k) + if (k > maxk) + maxk = k; + tree vectype = SLP_TREE_VECTYPE (slp_node); + if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits) + || maxk >= (DR_GROUP_SIZE (group_info) & ~(nunits - 1))) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "BB vectorization with gaps at the end of " + "a load is not supported\n"); + return false; + } + } + + if (!perm_ok) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, + vect_location, + "unsupported load permutation\n"); + return false; + } + + *slp_perm = true; + } + return true; } /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing conditional operation STMT_INFO. When returning true, store the mask - in *MASK, the type of its definition in *MASK_DT_OUT, the type of the - vectorized mask in *MASK_VECTYPE_OUT and the SLP node corresponding - to the mask in *MASK_NODE if MASK_NODE is not NULL. */ + in *MASK_NODE, the type of its definition in *MASK_DT_OUT and the type of + the vectorized mask in *MASK_VECTYPE_OUT. */ static bool -vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info, +vect_check_scalar_mask (vec_info *vinfo, slp_tree slp_node, unsigned mask_index, - tree *mask, slp_tree *mask_node, + slp_tree *mask_node, vect_def_type *mask_dt_out, tree *mask_vectype_out) { enum vect_def_type mask_dt; tree mask_vectype; slp_tree mask_node_1; - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, mask_index, - mask, &mask_node_1, &mask_dt, &mask_vectype)) + tree mask_; + if (!vect_is_simple_use (vinfo, slp_node, mask_index, + &mask_, &mask_node_1, &mask_dt, &mask_vectype)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -2773,7 +2768,7 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info, } if ((mask_dt == vect_constant_def || mask_dt == vect_external_def) - && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (*mask))) + && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask_))) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -2781,19 +2776,7 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info, return false; } - /* If the caller is not prepared for adjusting an external/constant - SLP mask vector type fail. */ - if (slp_node - && !mask_node - && SLP_TREE_DEF_TYPE (mask_node_1) != vect_internal_def) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "SLP mask argument is not vectorized.\n"); - return false; - } - - tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vectype = SLP_TREE_VECTYPE (slp_node); if (!mask_vectype) mask_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (vectype), mask_node_1); @@ -2820,11 +2803,11 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info, *mask_dt_out = mask_dt; *mask_vectype_out = mask_vectype; - if (mask_node) - *mask_node = mask_node_1; + *mask_node = mask_node_1; return true; } + /* Return true if stored value is suitable for vectorizing store statement STMT_INFO. When returning true, store the scalar stored in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT, @@ -2833,7 +2816,7 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info, static bool vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info, - slp_tree slp_node, tree *rhs, slp_tree *rhs_node, + slp_tree slp_node, slp_tree *rhs_node, vect_def_type *rhs_dt_out, tree *rhs_vectype_out, vec_load_store_type *vls_type_out) { @@ -2844,14 +2827,14 @@ vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info, && internal_store_fn_p (gimple_call_internal_fn (call))) op_no = internal_fn_stored_value_index (gimple_call_internal_fn (call)); } - if (slp_node) - op_no = vect_slp_child_index_for_operand - (stmt_info->stmt, op_no, STMT_VINFO_GATHER_SCATTER_P (stmt_info)); + op_no = vect_slp_child_index_for_operand + (stmt_info->stmt, op_no, STMT_VINFO_GATHER_SCATTER_P (stmt_info)); enum vect_def_type rhs_dt; tree rhs_vectype; - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, op_no, - rhs, rhs_node, &rhs_dt, &rhs_vectype)) + tree rhs; + if (!vect_is_simple_use (vinfo, slp_node, op_no, + &rhs, rhs_node, &rhs_dt, &rhs_vectype)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -2862,7 +2845,7 @@ vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info, /* In the case this is a store from a constant make sure native_encode_expr can handle it. */ if (rhs_dt == vect_constant_def - && CONSTANT_CLASS_P (*rhs) && native_encode_expr (*rhs, NULL, 64) == 0) + && CONSTANT_CLASS_P (rhs) && native_encode_expr (rhs, NULL, 64) == 0) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -2870,7 +2853,7 @@ vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info, return false; } - tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vectype = SLP_TREE_VECTYPE (slp_node); if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype)) { if (dump_enabled_p ()) @@ -2976,13 +2959,12 @@ vect_get_mask_load_else (int elsval, tree type) static gimple * vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info, - gimple_stmt_iterator *gsi, - gather_scatter_info *gs_info, + slp_tree slp_node, tree vectype, + gimple_stmt_iterator *gsi, tree decl, tree ptr, tree offset, tree mask) { - tree vectype = STMT_VINFO_VECTYPE (stmt_info); - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info->decl)); - tree rettype = TREE_TYPE (TREE_TYPE (gs_info->decl)); + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl)); + tree rettype = TREE_TYPE (TREE_TYPE (decl)); tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); /* ptrtype */ arglist = TREE_CHAIN (arglist); tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); @@ -3048,8 +3030,8 @@ vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info, mask_op = vect_build_all_ones_mask (vinfo, stmt_info, masktype); } - tree scale = build_int_cst (scaletype, gs_info->scale); - gimple *new_stmt = gimple_build_call (gs_info->decl, 5, src_op, ptr, op, + tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node)); + gimple *new_stmt = gimple_build_call (decl, 5, src_op, ptr, op, mask_op, scale); if (!useless_type_conversion_p (vectype, rettype)) @@ -3075,12 +3057,13 @@ vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info, static gimple * vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info, + slp_tree slp_node, gimple_stmt_iterator *gsi, - gather_scatter_info *gs_info, + tree decl, tree ptr, tree offset, tree oprnd, tree mask) { - tree rettype = TREE_TYPE (TREE_TYPE (gs_info->decl)); - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info->decl)); + tree rettype = TREE_TYPE (TREE_TYPE (decl)); + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl)); /* tree ptrtype = TREE_VALUE (arglist); */ arglist = TREE_CHAIN (arglist); tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); @@ -3144,9 +3127,9 @@ vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info, op = var; } - tree scale = build_int_cst (scaletype, gs_info->scale); + tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node)); gcall *new_stmt - = gimple_build_call (gs_info->decl, 5, ptr, mask_arg, op, src, scale); + = gimple_build_call (decl, 5, ptr, mask_arg, op, src, scale); return new_stmt; } @@ -3157,13 +3140,12 @@ vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info, containing loop. */ static void -vect_get_gather_scatter_ops (loop_vec_info loop_vinfo, - class loop *loop, stmt_vec_info stmt_info, - slp_tree slp_node, gather_scatter_info *gs_info, +vect_get_gather_scatter_ops (class loop *loop, slp_tree slp_node, tree *dataref_ptr, vec<tree> *vec_offset) { gimple_seq stmts = NULL; - *dataref_ptr = force_gimple_operand (gs_info->base, &stmts, true, NULL_TREE); + *dataref_ptr = force_gimple_operand (SLP_TREE_GS_BASE (slp_node), + &stmts, true, NULL_TREE); if (stmts != NULL) { basic_block new_bb; @@ -3171,16 +3153,7 @@ vect_get_gather_scatter_ops (loop_vec_info loop_vinfo, new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts); gcc_assert (!new_bb); } - if (slp_node) - vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_offset); - else - { - unsigned ncopies - = vect_get_num_copies (loop_vinfo, gs_info->offset_vectype); - vect_get_vec_defs_for_operand (loop_vinfo, stmt_info, ncopies, - gs_info->offset, vec_offset, - gs_info->offset_vectype); - } + vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_offset); } /* Prepare to implement a grouped or strided load or store using @@ -3193,15 +3166,14 @@ vect_get_gather_scatter_ops (loop_vec_info loop_vinfo, I * DR_STEP / SCALE. */ static void -vect_get_strided_load_store_ops (stmt_vec_info stmt_info, +vect_get_strided_load_store_ops (stmt_vec_info stmt_info, slp_tree node, + tree vectype, tree offset_vectype, loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi, - gather_scatter_info *gs_info, tree *dataref_bump, tree *vec_offset, vec_loop_lens *loop_lens) { struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); - tree vectype = STMT_VINFO_VECTYPE (stmt_info); if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)) { @@ -3238,15 +3210,15 @@ vect_get_strided_load_store_ops (stmt_vec_info stmt_info, /* The offset given in GS_INFO can have pointer type, so use the element type of the vector instead. */ - tree offset_type = TREE_TYPE (gs_info->offset_vectype); + tree offset_type = TREE_TYPE (offset_vectype); /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */ tree step = size_binop (EXACT_DIV_EXPR, unshare_expr (DR_STEP (dr)), - ssize_int (gs_info->scale)); + ssize_int (SLP_TREE_GS_SCALE (node))); step = fold_convert (offset_type, step); /* Create {0, X, X*2, X*3, ...}. */ - tree offset = fold_build2 (VEC_SERIES_EXPR, gs_info->offset_vectype, + tree offset = fold_build2 (VEC_SERIES_EXPR, offset_vectype, build_zero_cst (offset_type), step); *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo, offset); } @@ -3269,7 +3241,7 @@ vect_get_loop_variant_data_ptr_increment ( tree step = vect_dr_behavior (vinfo, dr_info)->step; /* gather/scatter never reach here. */ - gcc_assert (memory_access_type != VMAT_GATHER_SCATTER); + gcc_assert (!mat_gather_scatter_p (memory_access_type)); /* When we support SELECT_VL pattern, we dynamic adjust the memory address by .SELECT_VL result. @@ -3303,7 +3275,7 @@ static tree vect_get_data_ptr_increment (vec_info *vinfo, gimple_stmt_iterator *gsi, dr_vec_info *dr_info, tree aggr_type, vect_memory_access_type memory_access_type, - vec_loop_lens *loop_lens = nullptr) + vec_loop_lens *loop_lens) { if (memory_access_type == VMAT_INVARIANT) return size_zero_node; @@ -3326,29 +3298,17 @@ vect_get_data_ptr_increment (vec_info *vinfo, gimple_stmt_iterator *gsi, static bool vectorizable_bswap (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - gimple **vec_stmt, slp_tree slp_node, + slp_tree slp_node, slp_tree *slp_op, tree vectype_in, stmt_vector_for_cost *cost_vec) { tree op, vectype; gcall *stmt = as_a <gcall *> (stmt_info->stmt); - loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); - unsigned ncopies; op = gimple_call_arg (stmt, 0); - vectype = STMT_VINFO_VECTYPE (stmt_info); + vectype = SLP_TREE_VECTYPE (slp_node); poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); - /* Multiple types in SLP are handled by creating the appropriate number of - vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in - case of SLP. */ - if (slp_node) - ncopies = 1; - else - ncopies = vect_get_num_copies (loop_vinfo, vectype); - - gcc_assert (ncopies >= 1); - if (TYPE_SIZE (vectype_in) != TYPE_SIZE (vectype)) { if (dump_enabled_p ()) @@ -3378,10 +3338,9 @@ vectorizable_bswap (vec_info *vinfo, if (!can_vec_perm_const_p (vmode, vmode, indices)) return false; - if (! vec_stmt) + if (cost_vec) { - if (slp_node - && !vect_maybe_update_slp_op_vectype (slp_op[0], vectype_in)) + if (!vect_maybe_update_slp_op_vectype (slp_op[0], vectype_in)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -3389,14 +3348,13 @@ vectorizable_bswap (vec_info *vinfo, return false; } - STMT_VINFO_TYPE (stmt_info) = call_vec_info_type; + SLP_TREE_TYPE (slp_node) = call_vec_info_type; DUMP_VECT_SCOPE ("vectorizable_bswap"); record_stmt_cost (cost_vec, - 1, vector_stmt, stmt_info, 0, vect_prologue); + 1, vector_stmt, slp_node, 0, vect_prologue); record_stmt_cost (cost_vec, - slp_node - ? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) : ncopies, - vec_perm, stmt_info, 0, vect_body); + vect_get_num_copies (vinfo, slp_node), + vec_perm, slp_node, 0, vect_body); return true; } @@ -3404,8 +3362,7 @@ vectorizable_bswap (vec_info *vinfo, /* Transform. */ vec<tree> vec_oprnds = vNULL; - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, - op, &vec_oprnds); + vect_get_vec_defs (vinfo, slp_node, op, &vec_oprnds); /* Arguments are ready. create the new vector stmt. */ unsigned i; tree vop; @@ -3424,15 +3381,9 @@ vectorizable_bswap (vec_info *vinfo, new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR, vectype, tem2)); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - if (slp_node) - slp_node->push_vec_def (new_stmt); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + slp_node->push_vec_def (new_stmt); } - if (!slp_node) - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - vec_oprnds.release (); return true; } @@ -3465,21 +3416,22 @@ simple_integer_narrowing (tree vectype_out, tree vectype_in, /* Function vectorizable_call. Check if STMT_INFO performs a function call that can be vectorized. - If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized - stmt to replace it, put it in VEC_STMT, and insert it at GSI. + If COST_VEC is passed, calculate costs but don't change anything, + otherwise, vectorize STMT_INFO: create a vectorized stmt to replace + it, and insert it at GSI. Return true if STMT_INFO is vectorizable in this way. */ static bool vectorizable_call (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - gimple **vec_stmt, slp_tree slp_node, + slp_tree slp_node, stmt_vector_for_cost *cost_vec) { gcall *stmt; tree vec_dest; tree scalar_dest; tree op; - tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE; + tree vec_oprnd0 = NULL_TREE; tree vectype_out, vectype_in; poly_uint64 nunits_in; poly_uint64 nunits_out; @@ -3491,19 +3443,16 @@ vectorizable_call (vec_info *vinfo, vect_unknown_def_type }; tree vectypes[ARRAY_SIZE (dt)] = {}; slp_tree slp_op[ARRAY_SIZE (dt)] = {}; - int ndts = ARRAY_SIZE (dt); - int ncopies, j; auto_vec<tree, 8> vargs; enum { NARROW, NONE, WIDEN } modifier; size_t i, nargs; - tree lhs; tree clz_ctz_arg1 = NULL_TREE; if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo) return false; if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def - && ! vec_stmt) + && cost_vec) return false; /* Is STMT_INFO a vectorizable call? */ @@ -3523,7 +3472,7 @@ vectorizable_call (vec_info *vinfo, gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt)); - vectype_out = STMT_VINFO_VECTYPE (stmt_info); + vectype_out = SLP_TREE_VECTYPE (slp_node); /* Process function arguments. */ rhs_type = NULL_TREE; @@ -3554,19 +3503,25 @@ vectorizable_call (vec_info *vinfo, int mask_opno = -1; if (internal_fn_p (cfn)) - mask_opno = internal_fn_mask_index (as_internal_fn (cfn)); + { + /* We can only handle direct internal masked calls here, + vectorizable_simd_clone_call is for the rest. */ + if (cfn == CFN_MASK_CALL) + return false; + mask_opno = internal_fn_mask_index (as_internal_fn (cfn)); + } for (i = 0; i < nargs; i++) { if ((int) i == mask_opno) { - if (!vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_opno, - &op, &slp_op[i], &dt[i], &vectypes[i])) + if (!vect_check_scalar_mask (vinfo, slp_node, mask_opno, + &slp_op[i], &dt[i], &vectypes[i])) return false; continue; } - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, + if (!vect_is_simple_use (vinfo, slp_node, i, &op, &slp_op[i], &dt[i], &vectypes[i])) { if (dump_enabled_p ()) @@ -3602,7 +3557,7 @@ vectorizable_call (vec_info *vinfo, from the scalar type. */ if (!vectype_in) vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node); - if (vec_stmt) + if (!cost_vec) gcc_assert (vectype_in); if (!vectype_in) { @@ -3622,13 +3577,14 @@ vectorizable_call (vec_info *vinfo, return false; } - if (vect_emulated_vector_p (vectype_in) || vect_emulated_vector_p (vectype_out)) - { + if (vect_emulated_vector_p (vectype_in) + || vect_emulated_vector_p (vectype_out)) + { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, "use emulated vector type for call\n"); return false; - } + } /* FORNOW */ nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in); @@ -3683,7 +3639,7 @@ vectorizable_call (vec_info *vinfo, if (ifn == IFN_LAST && !fndecl) { if (cfn == CFN_GOMP_SIMD_LANE - && (!slp_node || SLP_TREE_LANES (slp_node) == 1) + && SLP_TREE_LANES (slp_node) == 1 && loop_vinfo && LOOP_VINFO_LOOP (loop_vinfo)->simduid && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME @@ -3699,7 +3655,7 @@ vectorizable_call (vec_info *vinfo, || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP32) || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP64) || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP128))) - return vectorizable_bswap (vinfo, stmt_info, gsi, vec_stmt, slp_node, + return vectorizable_bswap (vinfo, stmt_info, gsi, slp_node, slp_op, vectype_in, cost_vec); else { @@ -3710,43 +3666,29 @@ vectorizable_call (vec_info *vinfo, } } - if (slp_node) - ncopies = 1; - else if (modifier == NARROW && ifn == IFN_LAST) - ncopies = vect_get_num_copies (loop_vinfo, vectype_out); - else - ncopies = vect_get_num_copies (loop_vinfo, vectype_in); - - /* Sanity check: make sure that at least one copy of the vectorized stmt - needs to be generated. */ - gcc_assert (ncopies >= 1); - - int reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info); - internal_fn cond_fn = get_conditional_internal_fn (ifn); + int reduc_idx = SLP_TREE_REDUC_IDX (slp_node); + internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1 + ? ifn : get_conditional_internal_fn (ifn)); internal_fn cond_len_fn = get_len_internal_fn (ifn); int len_opno = internal_fn_len_index (cond_len_fn); vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL); vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL); - if (!vec_stmt) /* transformation not required. */ + unsigned int nvectors = vect_get_num_copies (vinfo, slp_node); + if (cost_vec) /* transformation not required. */ { - if (slp_node) - for (i = 0; i < nargs; ++i) - if (!vect_maybe_update_slp_op_vectype (slp_op[i], - vectypes[i] - ? vectypes[i] : vectype_in)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "incompatible vector types for invariants\n"); - return false; - } - STMT_VINFO_TYPE (stmt_info) = call_vec_info_type; + for (i = 0; i < nargs; ++i) + if (!vect_maybe_update_slp_op_vectype (slp_op[i], + vectypes[i] + ? vectypes[i] : vectype_in)) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "incompatible vector types for invariants\n"); + return false; + } + SLP_TREE_TYPE (slp_node) = call_vec_info_type; DUMP_VECT_SCOPE ("vectorizable_call"); - vect_model_simple_cost (vinfo, stmt_info, - ncopies, dt, ndts, slp_node, cost_vec); - if (ifn != IFN_LAST && modifier == NARROW && !slp_node) - record_stmt_cost (cost_vec, ncopies / 2, - vec_promote_demote, stmt_info, 0, vect_body); + vect_model_simple_cost (vinfo, 1, slp_node, cost_vec); if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) @@ -3768,10 +3710,6 @@ vectorizable_call (vec_info *vinfo, } else { - unsigned int nvectors - = (slp_node - ? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) - : ncopies); tree scalar_mask = NULL_TREE; if (mask_opno >= 0) scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno); @@ -3811,7 +3749,7 @@ vectorizable_call (vec_info *vinfo, else if (reduc_idx >= 0) gcc_unreachable (); } - else if (masked_loop_p && reduc_idx >= 0) + else if (masked_loop_p && mask_opno == -1 && reduc_idx >= 0) { ifn = cond_fn; vect_nargs += 2; @@ -3824,220 +3762,118 @@ vectorizable_call (vec_info *vinfo, tree prev_res = NULL_TREE; vargs.safe_grow (vect_nargs, true); auto_vec<vec<tree> > vec_defs (nargs); - for (j = 0; j < ncopies; ++j) + + /* Build argument list for the vectorized call. */ + if (cfn == CFN_GOMP_SIMD_LANE) + { + for (i = 0; i < nvectors; ++i) + { + /* ??? For multi-lane SLP we'd need to build + { 0, 0, .., 1, 1, ... }. */ + tree cst = build_index_vector (vectype_out, + i * nunits_out, 1); + tree new_var + = vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_"); + gimple *init_stmt = gimple_build_assign (new_var, cst); + vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL); + new_temp = make_ssa_name (vec_dest); + gimple *new_stmt = gimple_build_assign (new_temp, new_var); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + slp_node->push_vec_def (new_stmt); + } + } + else { - /* Build argument list for the vectorized call. */ - if (slp_node) + vec<tree> vec_oprnds0; + vect_get_slp_defs (vinfo, slp_node, &vec_defs); + vec_oprnds0 = vec_defs[0]; + + /* Arguments are ready. Create the new vector stmt. */ + FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0) { - if (cfn == CFN_GOMP_SIMD_LANE) + int varg = 0; + /* Add the mask if necessary. */ + if (masked_loop_p && mask_opno == -1 && reduc_idx >= 0) + { + gcc_assert (internal_fn_mask_index (ifn) == varg); + unsigned int vec_num = vec_oprnds0.length (); + vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi, masks, + vec_num, vectype_out, i); + } + size_t k; + for (k = 0; k < nargs; k++) + { + vec<tree> vec_oprndsk = vec_defs[k]; + vargs[varg++] = vec_oprndsk[i]; + } + /* Add the else value if necessary. */ + if (masked_loop_p && mask_opno == -1 && reduc_idx >= 0) + { + gcc_assert (internal_fn_else_index (ifn) == varg); + vargs[varg++] = vargs[reduc_idx + 1]; + } + if (clz_ctz_arg1) + vargs[varg++] = clz_ctz_arg1; + + gimple *new_stmt; + if (modifier == NARROW) { - for (i = 0; i < SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); ++i) + /* We don't define any narrowing conditional functions + at present. */ + gcc_assert (mask_opno < 0); + tree half_res = make_ssa_name (vectype_in); + gcall *call = gimple_build_call_internal_vec (ifn, vargs); + gimple_call_set_lhs (call, half_res); + gimple_call_set_nothrow (call, true); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + if ((i & 1) == 0) { - /* ??? For multi-lane SLP we'd need to build - { 0, 0, .., 1, 1, ... }. */ - tree cst = build_index_vector (vectype_out, - i * nunits_out, 1); - tree new_var - = vect_get_new_ssa_name (vectype_out, vect_simple_var, - "cst_"); - gimple *init_stmt = gimple_build_assign (new_var, cst); - vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL); - new_temp = make_ssa_name (vec_dest); - gimple *new_stmt - = gimple_build_assign (new_temp, new_var); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, - gsi); - slp_node->push_vec_def (new_stmt); + prev_res = half_res; + continue; } - continue; + new_temp = make_ssa_name (vec_dest); + new_stmt = vect_gimple_build (new_temp, convert_code, + prev_res, half_res); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } - - vec<tree> vec_oprnds0; - vect_get_slp_defs (vinfo, slp_node, &vec_defs); - vec_oprnds0 = vec_defs[0]; - - /* Arguments are ready. Create the new vector stmt. */ - FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0) + else { - int varg = 0; - if (masked_loop_p && reduc_idx >= 0) + if (len_opno >= 0 && len_loop_p) { unsigned int vec_num = vec_oprnds0.length (); - /* Always true for SLP. */ - gcc_assert (ncopies == 1); - vargs[varg++] = vect_get_loop_mask (loop_vinfo, - gsi, masks, vec_num, - vectype_out, i); + tree len = vect_get_loop_len (loop_vinfo, gsi, lens, + vec_num, vectype_out, i, 1); + signed char biasval + = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); + tree bias = build_int_cst (intQI_type_node, biasval); + vargs[len_opno] = len; + vargs[len_opno + 1] = bias; } - size_t k; - for (k = 0; k < nargs; k++) + else if (mask_opno >= 0 && masked_loop_p) { - vec<tree> vec_oprndsk = vec_defs[k]; - vargs[varg++] = vec_oprndsk[i]; + unsigned int vec_num = vec_oprnds0.length (); + tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks, + vec_num, vectype_out, i); + vargs[mask_opno] + = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask, + vargs[mask_opno], gsi); } - if (masked_loop_p && reduc_idx >= 0) - vargs[varg++] = vargs[reduc_idx + 1]; - if (clz_ctz_arg1) - vargs[varg++] = clz_ctz_arg1; - gimple *new_stmt; - if (modifier == NARROW) - { - /* We don't define any narrowing conditional functions - at present. */ - gcc_assert (mask_opno < 0); - tree half_res = make_ssa_name (vectype_in); - gcall *call - = gimple_build_call_internal_vec (ifn, vargs); - gimple_call_set_lhs (call, half_res); - gimple_call_set_nothrow (call, true); - vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); - if ((i & 1) == 0) - { - prev_res = half_res; - continue; - } - new_temp = make_ssa_name (vec_dest); - new_stmt = vect_gimple_build (new_temp, convert_code, - prev_res, half_res); - vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); - } + gcall *call; + if (ifn != IFN_LAST) + call = gimple_build_call_internal_vec (ifn, vargs); else - { - if (len_opno >= 0 && len_loop_p) - { - unsigned int vec_num = vec_oprnds0.length (); - /* Always true for SLP. */ - gcc_assert (ncopies == 1); - tree len - = vect_get_loop_len (loop_vinfo, gsi, lens, vec_num, - vectype_out, i, 1); - signed char biasval - = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); - tree bias = build_int_cst (intQI_type_node, biasval); - vargs[len_opno] = len; - vargs[len_opno + 1] = bias; - } - else if (mask_opno >= 0 && masked_loop_p) - { - unsigned int vec_num = vec_oprnds0.length (); - /* Always true for SLP. */ - gcc_assert (ncopies == 1); - tree mask = vect_get_loop_mask (loop_vinfo, - gsi, masks, vec_num, - vectype_out, i); - vargs[mask_opno] = prepare_vec_mask - (loop_vinfo, TREE_TYPE (mask), mask, - vargs[mask_opno], gsi); - } - - gcall *call; - if (ifn != IFN_LAST) - call = gimple_build_call_internal_vec (ifn, vargs); - else - call = gimple_build_call_vec (fndecl, vargs); - new_temp = make_ssa_name (vec_dest, call); - gimple_call_set_lhs (call, new_temp); - gimple_call_set_nothrow (call, true); - vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); - new_stmt = call; - } - slp_node->push_vec_def (new_stmt); - } - continue; - } - - int varg = 0; - if (masked_loop_p && reduc_idx >= 0) - vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies, - vectype_out, j); - for (i = 0; i < nargs; i++) - { - op = gimple_call_arg (stmt, i); - if (j == 0) - { - vec_defs.quick_push (vNULL); - vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies, - op, &vec_defs[i], - vectypes[i]); + call = gimple_build_call_vec (fndecl, vargs); + new_temp = make_ssa_name (vec_dest, call); + gimple_call_set_lhs (call, new_temp); + gimple_call_set_nothrow (call, true); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; } - vargs[varg++] = vec_defs[i][j]; + slp_node->push_vec_def (new_stmt); } - if (masked_loop_p && reduc_idx >= 0) - vargs[varg++] = vargs[reduc_idx + 1]; - if (clz_ctz_arg1) - vargs[varg++] = clz_ctz_arg1; - - if (len_opno >= 0 && len_loop_p) - { - tree len = vect_get_loop_len (loop_vinfo, gsi, lens, ncopies, - vectype_out, j, 1); - signed char biasval - = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); - tree bias = build_int_cst (intQI_type_node, biasval); - vargs[len_opno] = len; - vargs[len_opno + 1] = bias; - } - else if (mask_opno >= 0 && masked_loop_p) - { - tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies, - vectype_out, j); - vargs[mask_opno] - = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask, - vargs[mask_opno], gsi); - } - - gimple *new_stmt; - if (cfn == CFN_GOMP_SIMD_LANE) - { - tree cst = build_index_vector (vectype_out, j * nunits_out, 1); - tree new_var - = vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_"); - gimple *init_stmt = gimple_build_assign (new_var, cst); - vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL); - new_temp = make_ssa_name (vec_dest); - new_stmt = gimple_build_assign (new_temp, new_var); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - } - else if (modifier == NARROW) - { - /* We don't define any narrowing conditional functions at - present. */ - gcc_assert (mask_opno < 0); - tree half_res = make_ssa_name (vectype_in); - gcall *call = gimple_build_call_internal_vec (ifn, vargs); - gimple_call_set_lhs (call, half_res); - gimple_call_set_nothrow (call, true); - vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); - if ((j & 1) == 0) - { - prev_res = half_res; - continue; - } - new_temp = make_ssa_name (vec_dest); - new_stmt = vect_gimple_build (new_temp, convert_code, prev_res, - half_res); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - } - else - { - gcall *call; - if (ifn != IFN_LAST) - call = gimple_build_call_internal_vec (ifn, vargs); - else - call = gimple_build_call_vec (fndecl, vargs); - new_temp = make_ssa_name (vec_dest, call); - gimple_call_set_lhs (call, new_temp); - gimple_call_set_nothrow (call, true); - vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); - new_stmt = call; - } - - if (j == (modifier == NARROW ? 1 : 0)) - *vec_stmt = new_stmt; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } + for (i = 0; i < nargs; i++) { vec<tree> vec_oprndsi = vec_defs[i]; @@ -4049,73 +3885,36 @@ vectorizable_call (vec_info *vinfo, auto_vec<vec<tree> > vec_defs (nargs); /* We don't define any narrowing conditional functions at present. */ gcc_assert (mask_opno < 0); - for (j = 0; j < ncopies; ++j) - { - /* Build argument list for the vectorized call. */ - if (j == 0) - vargs.create (nargs * 2); - else - vargs.truncate (0); - - if (slp_node) - { - vec<tree> vec_oprnds0; - vect_get_slp_defs (vinfo, slp_node, &vec_defs); - vec_oprnds0 = vec_defs[0]; + /* Build argument list for the vectorized call. */ + vargs.create (nargs * 2); - /* Arguments are ready. Create the new vector stmt. */ - for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2) - { - size_t k; - vargs.truncate (0); - for (k = 0; k < nargs; k++) - { - vec<tree> vec_oprndsk = vec_defs[k]; - vargs.quick_push (vec_oprndsk[i]); - vargs.quick_push (vec_oprndsk[i + 1]); - } - gcall *call; - if (ifn != IFN_LAST) - call = gimple_build_call_internal_vec (ifn, vargs); - else - call = gimple_build_call_vec (fndecl, vargs); - new_temp = make_ssa_name (vec_dest, call); - gimple_call_set_lhs (call, new_temp); - gimple_call_set_nothrow (call, true); - vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); - slp_node->push_vec_def (call); - } - continue; - } + vect_get_slp_defs (vinfo, slp_node, &vec_defs); + vec<tree> vec_oprnds0 = vec_defs[0]; - for (i = 0; i < nargs; i++) + /* Arguments are ready. Create the new vector stmt. */ + for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2) + { + size_t k; + vargs.truncate (0); + for (k = 0; k < nargs; k++) { - op = gimple_call_arg (stmt, i); - if (j == 0) - { - vec_defs.quick_push (vNULL); - vect_get_vec_defs_for_operand (vinfo, stmt_info, 2 * ncopies, - op, &vec_defs[i], vectypes[i]); - } - vec_oprnd0 = vec_defs[i][2*j]; - vec_oprnd1 = vec_defs[i][2*j+1]; - - vargs.quick_push (vec_oprnd0); - vargs.quick_push (vec_oprnd1); + vec<tree> vec_oprndsk = vec_defs[k]; + vargs.quick_push (vec_oprndsk[i]); + vargs.quick_push (vec_oprndsk[i + 1]); } - - gcall *new_stmt = gimple_build_call_vec (fndecl, vargs); - new_temp = make_ssa_name (vec_dest, new_stmt); - gimple_call_set_lhs (new_stmt, new_temp); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + gcall *call; + if (ifn != IFN_LAST) + call = gimple_build_call_internal_vec (ifn, vargs); + else + call = gimple_build_call_vec (fndecl, vargs); + new_temp = make_ssa_name (vec_dest, call); + gimple_call_set_lhs (call, new_temp); + gimple_call_set_nothrow (call, true); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + slp_node->push_vec_def (call); } - if (!slp_node) - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - for (i = 0; i < nargs; i++) { vec<tree> vec_oprndsi = vec_defs[i]; @@ -4128,21 +3927,6 @@ vectorizable_call (vec_info *vinfo, vargs.release (); - /* The call in STMT might prevent it from being removed in dce. - We however cannot remove it here, due to the way the ssa name - it defines is mapped to the new definition. So just replace - rhs of the statement with something harmless. */ - - if (slp_node) - return true; - - stmt_info = vect_orig_stmt (stmt_info); - lhs = gimple_get_lhs (stmt_info->stmt); - - gassign *new_stmt - = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs))); - vinfo->replace_stmt (gsi, stmt_info, new_stmt); - return true; } @@ -4229,19 +4013,19 @@ vect_simd_lane_linear (tree op, class loop *loop, Check if STMT_INFO performs a function call that can be vectorized by calling a simd clone of the function. - If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized - stmt to replace it, put it in VEC_STMT, and insert it at GSI. + If COST_VEC is passed, calculate costs but don't change anything, + otherwise, vectorize STMT_INFO: create a vectorized stmt to replace + it, and insert it at GSI. Return true if STMT_INFO is vectorizable in this way. */ static bool vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - gimple **vec_stmt, slp_tree slp_node, - stmt_vector_for_cost *) + slp_tree slp_node, + stmt_vector_for_cost *cost_vec) { tree vec_dest; tree scalar_dest; - tree op, type; tree vec_oprnd0 = NULL_TREE; tree vectype; poly_uint64 nunits; @@ -4249,11 +4033,11 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo); class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL; tree fndecl, new_temp; - int ncopies, j; + int j; auto_vec<simd_call_arg_info> arginfo; vec<tree> vargs = vNULL; size_t i, nargs; - tree lhs, rtype, ratype; + tree rtype, ratype; vec<constructor_elt, va_gc> *ret_ctor_elts = NULL; int masked_call_offset = 0; @@ -4283,7 +4067,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, return false; if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def - && ! vec_stmt) + && cost_vec) return false; if (gimple_call_lhs (stmt) @@ -4292,7 +4076,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt)); - vectype = STMT_VINFO_VECTYPE (stmt_info); + vectype = SLP_TREE_VECTYPE (slp_node); if (loop_vinfo && nested_in_vect_loop_p (loop, stmt_info)) return false; @@ -4304,10 +4088,9 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, if (nargs == 0) return false; - vec<tree>& simd_clone_info = (slp_node ? SLP_TREE_SIMD_CLONE_INFO (slp_node) - : STMT_VINFO_SIMD_CLONE_INFO (stmt_info)); - if (!vec_stmt) - simd_clone_info.truncate (0); + vect_simd_clone_data _data; + vect_simd_clone_data &data = slp_node->get_data (_data); + vec<tree>& simd_clone_info = data.simd_clone_info; arginfo.reserve (nargs, true); auto_vec<slp_tree> slp_op; slp_op.safe_grow_cleared (nargs); @@ -4316,16 +4099,17 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, { simd_call_arg_info thisarginfo; affine_iv iv; + tree op; thisarginfo.linear_step = 0; thisarginfo.align = 0; thisarginfo.op = NULL_TREE; thisarginfo.simd_lane_linear = false; - int op_no = i + masked_call_offset; - if (slp_node) - op_no = vect_slp_child_index_for_operand (stmt, op_no, false); - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, + int op_no = vect_slp_child_index_for_operand (stmt, + i + masked_call_offset, + false); + if (!vect_is_simple_use (vinfo, slp_node, op_no, &op, &slp_op[i], &thisarginfo.dt, &thisarginfo.vectype) || thisarginfo.dt == vect_uninitialized_def) @@ -4343,10 +4127,9 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, at analysis time, handling conflicts via vect_maybe_update_slp_op_vectype. At transform time we have a vector type recorded for SLP. */ - gcc_assert (!vec_stmt - || !slp_node + gcc_assert (cost_vec || thisarginfo.vectype != NULL_TREE); - if (!vec_stmt) + if (cost_vec) thisarginfo.vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op), slp_node); @@ -4355,8 +4138,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, gcc_assert (thisarginfo.vectype != NULL_TREE); /* For linear arguments, the analyze phase should have saved - the base and step in {STMT_VINFO,SLP_TREE}_SIMD_CLONE_INFO. */ - if (vec_stmt + the base and step. */ + if (!cost_vec && i * 3 + 4 <= simd_clone_info.length () && simd_clone_info[i * 3 + 2]) { @@ -4380,10 +4163,11 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, thisarginfo.op, bias); } } - else if (!vec_stmt + else if (cost_vec && thisarginfo.dt != vect_constant_def && thisarginfo.dt != vect_external_def && loop_vinfo + && SLP_TREE_LANES (slp_node) == 1 && TREE_CODE (op) == SSA_NAME && simple_iv (loop, loop_containing_stmt (stmt), op, &iv, false) @@ -4394,28 +4178,35 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, } else if ((thisarginfo.dt == vect_constant_def || thisarginfo.dt == vect_external_def) + && SLP_TREE_LANES (slp_node) == 1 && POINTER_TYPE_P (TREE_TYPE (op))) thisarginfo.align = get_pointer_alignment (op) / BITS_PER_UNIT; /* Addresses of array elements indexed by GOMP_SIMD_LANE are linear too. */ - if (POINTER_TYPE_P (TREE_TYPE (op)) + if (SLP_TREE_LANES (slp_node) == 1 + && POINTER_TYPE_P (TREE_TYPE (op)) && !thisarginfo.linear_step - && !vec_stmt + && cost_vec && thisarginfo.dt != vect_constant_def && thisarginfo.dt != vect_external_def && loop_vinfo && TREE_CODE (op) == SSA_NAME) vect_simd_lane_linear (op, loop, &thisarginfo); + if (!vectype) + vectype = thisarginfo.vectype; arginfo.quick_push (thisarginfo); } poly_uint64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1; - unsigned group_size = slp_node ? SLP_TREE_LANES (slp_node) : 1; + unsigned group_size = SLP_TREE_LANES (slp_node); unsigned int badness = 0; + unsigned int badness_inbranch = 0; struct cgraph_node *bestn = NULL; - if (vec_stmt) - bestn = cgraph_node::get (simd_clone_info[0]); + struct cgraph_node *bestn_inbranch = NULL; + if (!cost_vec) + bestn = ((loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) + ? data.clone_inbranch : data.clone); else for (struct cgraph_node *n = node->simd_clones; n != NULL; n = n->simdclone->next_clone) @@ -4441,7 +4232,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, if (n->simdclone->inbranch) this_badness += 8192; - /* If STMT_VINFO_VECTYPE has not been set yet pass the general vector + /* If SLP_TREE_VECTYPE has not been set yet pass the general vector mode, which for targets that use it will determine what ISA we can vectorize this code with. */ machine_mode vector_mode = vinfo->vector_mode; @@ -4456,10 +4247,13 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, switch (n->simdclone->args[i].arg_type) { case SIMD_CLONE_ARG_TYPE_VECTOR: - if (!useless_type_conversion_p - (n->simdclone->args[i].orig_type, - TREE_TYPE (gimple_call_arg (stmt, - i + masked_call_offset)))) + if (VECTOR_BOOLEAN_TYPE_P (n->simdclone->args[i].vector_type)) + /* Vector mask arguments are not supported. */ + i = -1; + else if (!useless_type_conversion_p + (n->simdclone->args[i].orig_type, + TREE_TYPE (gimple_call_arg (stmt, + i + masked_call_offset)))) i = -1; else if (arginfo[i].dt == vect_constant_def || arginfo[i].dt == vect_external_def @@ -4467,8 +4261,9 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, this_badness += 64; break; case SIMD_CLONE_ARG_TYPE_UNIFORM: - if (arginfo[i].dt != vect_constant_def - && arginfo[i].dt != vect_external_def) + if ((arginfo[i].dt != vect_constant_def + && arginfo[i].dt != vect_external_def) + || SLP_TREE_LANES (slp_node) != 1) i = -1; break; case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP: @@ -4489,12 +4284,32 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, i = -1; break; case SIMD_CLONE_ARG_TYPE_MASK: + if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode) + && n->simdclone->mask_mode != VOIDmode) + i = -1; /* While we can create a traditional data vector from an incoming integer mode mask we have no good way to force generate an integer mode mask from a traditional boolean vector input. */ - if (SCALAR_INT_MODE_P (n->simdclone->mask_mode) - && !SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype))) + else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode) + && !SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype))) + i = -1; + else if (n->simdclone->mask_mode == VOIDmode + /* FORNOW we only have partial support for vector-type + masks that can't hold all of simdlen. */ + && (maybe_ne (TYPE_VECTOR_SUBPARTS (n->simdclone->args[i].vector_type), + TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)) + /* Verify we can compute the mask argument. */ + || !expand_vec_cond_expr_p (n->simdclone->args[i].vector_type, + arginfo[i].vectype))) + i = -1; + else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode) + /* FORNOW we only have partial support for + integer-type masks that represent the same number + of lanes as the vectorized mask inputs. */ + && maybe_ne (exact_div (n->simdclone->simdlen, + n->simdclone->args[i].linear_step), + TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))) i = -1; else if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode) && SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype))) @@ -4522,114 +4337,27 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, SIMD_CLONE_ARG_TYPE_MASK); /* Penalize using a masked SIMD clone in a non-masked loop, that is not in a branch, as we'd have to construct an all-true mask. */ - if (!loop_vinfo || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) - this_badness += 64; + this_badness += 64; } if (bestn == NULL || this_badness < badness) { bestn = n; badness = this_badness; } + if (n->simdclone->inbranch + && (bestn_inbranch == NULL || this_badness < badness_inbranch)) + { + bestn_inbranch = n; + badness_inbranch = this_badness; + } } if (bestn == NULL) return false; - unsigned int num_mask_args = 0; - if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode)) - for (i = 0; i < nargs; i++) - if (bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK) - num_mask_args++; - - for (i = 0; i < nargs; i++) - { - if ((arginfo[i].dt == vect_constant_def - || arginfo[i].dt == vect_external_def) - && bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR) - { - tree arg_type = TREE_TYPE (gimple_call_arg (stmt, - i + masked_call_offset)); - arginfo[i].vectype = get_vectype_for_scalar_type (vinfo, arg_type, - slp_node); - if (arginfo[i].vectype == NULL - || !constant_multiple_p (bestn->simdclone->simdlen, - TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))) - return false; - } - - if (bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR - && VECTOR_BOOLEAN_TYPE_P (bestn->simdclone->args[i].vector_type)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "vector mask arguments are not supported.\n"); - return false; - } - - if (bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK) - { - tree clone_arg_vectype = bestn->simdclone->args[i].vector_type; - if (bestn->simdclone->mask_mode == VOIDmode) - { - if (maybe_ne (TYPE_VECTOR_SUBPARTS (clone_arg_vectype), - TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))) - { - /* FORNOW we only have partial support for vector-type masks - that can't hold all of simdlen. */ - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, - vect_location, - "in-branch vector clones are not yet" - " supported for mismatched vector sizes.\n"); - return false; - } - if (!expand_vec_cond_expr_p (clone_arg_vectype, - arginfo[i].vectype)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, - vect_location, - "cannot compute mask argument for" - " in-branch vector clones.\n"); - return false; - } - } - else if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode)) - { - if (!SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)) - || maybe_ne (exact_div (bestn->simdclone->simdlen, - num_mask_args), - TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))) - { - /* FORNOW we only have partial support for integer-type masks - that represent the same number of lanes as the - vectorized mask inputs. */ - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, - vect_location, - "in-branch vector clones are not yet " - "supported for mismatched vector sizes.\n"); - return false; - } - } - else - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, - vect_location, - "in-branch vector clones not supported" - " on this target.\n"); - return false; - } - } - } - fndecl = bestn->decl; nunits = bestn->simdclone->simdlen; - if (slp_node) - ncopies = vector_unroll_factor (vf * group_size, nunits); - else - ncopies = vector_unroll_factor (vf, nunits); + int ncopies = vector_unroll_factor (vf * group_size, nunits); /* If the function isn't const, only allow it in simd loops where user has asserted that at least nunits consecutive iterations can be @@ -4638,21 +4366,36 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, && gimple_vuse (stmt)) return false; + /* ncopies is the number of SIMD clone calls we create, since simdlen + is not necessarily matching nunits of the vector types used, track + that in ncopies_in. */ + int ncopies_in = vect_get_num_vectors (vf * group_size, vectype); + /* Sanity check: make sure that at least one copy of the vectorized stmt needs to be generated. */ gcc_assert (ncopies >= 1); - if (!vec_stmt) /* transformation not required. */ + if (cost_vec) /* transformation not required. */ { - if (slp_node) - for (unsigned i = 0; i < nargs; ++i) - if (!vect_maybe_update_slp_op_vectype (slp_op[i], arginfo[i].vectype)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "incompatible vector types for invariants\n"); - return false; - } + for (unsigned i = 0; i < nargs; ++i) + if (!vect_maybe_update_slp_op_vectype (slp_op[i], arginfo[i].vectype)) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "incompatible vector types for invariants\n"); + return false; + } + + if (!bestn_inbranch && loop_vinfo) + { + if (dump_enabled_p () + && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)) + dump_printf_loc (MSG_NOTE, vect_location, + "can't use a fully-masked loop because no" + " masked simd clone was available.\n"); + LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false; + } + /* When the original call is pure or const but the SIMD ABI dictates an aggregate return we will have to use a virtual definition and in a loop eventually even need to add a virtual PHI. That's @@ -4664,54 +4407,79 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, /* ??? For SLP code-gen we end up inserting after the last vector argument def rather than at the original call position so automagic virtual operand updating doesn't work. */ - if (gimple_vuse (stmt) && slp_node) + if (gimple_vuse (stmt)) vinfo->any_known_not_updated_vssa = true; - simd_clone_info.safe_push (bestn->decl); - for (i = 0; i < bestn->simdclone->nargs; i++) + + data.clone = bestn; + data.clone_inbranch = bestn_inbranch; + + simd_clone_info.safe_push (NULL_TREE); + for (i = 0; + i < (bestn_inbranch ? bestn_inbranch : bestn)->simdclone->nargs; i++) { - switch (bestn->simdclone->args[i].arg_type) + if (loop_vinfo + && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) + && (bestn_inbranch->simdclone->args[i].arg_type + == SIMD_CLONE_ARG_TYPE_MASK)) { - default: - continue; - case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP: - case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP: - { - simd_clone_info.safe_grow_cleared (i * 3 + 1, true); - simd_clone_info.safe_push (arginfo[i].op); - tree lst = POINTER_TYPE_P (TREE_TYPE (arginfo[i].op)) - ? size_type_node : TREE_TYPE (arginfo[i].op); - tree ls = build_int_cst (lst, arginfo[i].linear_step); - simd_clone_info.safe_push (ls); - tree sll = arginfo[i].simd_lane_linear - ? boolean_true_node : boolean_false_node; - simd_clone_info.safe_push (sll); - } - break; - case SIMD_CLONE_ARG_TYPE_MASK: - if (loop_vinfo - && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)) + if (masked_call_offset) + /* When there is an explicit mask we require the + number of elements to match up. */ vect_record_loop_mask (loop_vinfo, &LOOP_VINFO_MASKS (loop_vinfo), - ncopies, vectype, op); - - break; + ncopies_in, vectype, NULL_TREE); + else + { + /* When there is no explicit mask on the call we have + more relaxed requirements. */ + tree masktype; + poly_uint64 callee_nelements; + if (SCALAR_INT_MODE_P (bestn_inbranch->simdclone->mask_mode)) + { + callee_nelements + = exact_div (bestn_inbranch->simdclone->simdlen, + bestn_inbranch->simdclone->args[i].linear_step); + masktype = get_related_vectype_for_scalar_type + (vinfo->vector_mode, TREE_TYPE (vectype), + callee_nelements); + } + else + { + masktype = bestn_inbranch->simdclone->args[i].vector_type; + callee_nelements = TYPE_VECTOR_SUBPARTS (masktype); + } + auto o = vector_unroll_factor (nunits, callee_nelements); + vect_record_loop_mask (loop_vinfo, + &LOOP_VINFO_MASKS (loop_vinfo), + ncopies * o, masktype, NULL_TREE); + } } - } - - if (!bestn->simdclone->inbranch && loop_vinfo) - { - if (dump_enabled_p () - && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)) - dump_printf_loc (MSG_NOTE, vect_location, - "can't use a fully-masked loop because a" - " non-masked simd clone was selected.\n"); - LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false; - } - - STMT_VINFO_TYPE (stmt_info) = call_simd_clone_vec_info_type; + else if ((bestn->simdclone->args[i].arg_type + == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP) + || (bestn->simdclone->args[i].arg_type + == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP) + || (bestn_inbranch + && ((bestn_inbranch->simdclone->args[i].arg_type + == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP) + || (bestn_inbranch->simdclone->args[i].arg_type + == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)))) + { + simd_clone_info.safe_grow_cleared (i * 3 + 1, true); + simd_clone_info.safe_push (arginfo[i].op); + tree lst = (POINTER_TYPE_P (TREE_TYPE (arginfo[i].op)) + ? size_type_node : TREE_TYPE (arginfo[i].op)); + tree ls = build_int_cst (lst, arginfo[i].linear_step); + simd_clone_info.safe_push (ls); + tree sll = (arginfo[i].simd_lane_linear + ? boolean_true_node : boolean_false_node); + simd_clone_info.safe_push (sll); + } + } + + SLP_TREE_TYPE (slp_node) = call_simd_clone_vec_info_type; + slp_node->data = new vect_simd_clone_data (std::move (_data)); DUMP_VECT_SCOPE ("vectorizable_simd_clone_call"); -/* vect_model_simple_cost (vinfo, stmt_info, ncopies, - dt, slp_node, cost_vec); */ +/* vect_model_simple_cost (vinfo, 1, slp_node, cost_vec); */ return true; } @@ -4739,13 +4507,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, auto_vec<vec<tree> > vec_oprnds; auto_vec<unsigned> vec_oprnds_i; vec_oprnds_i.safe_grow_cleared (nargs, true); - if (slp_node) - { - vec_oprnds.reserve_exact (nargs); - vect_get_slp_defs (vinfo, slp_node, &vec_oprnds); - } - else - vec_oprnds.safe_grow_cleared (nargs, true); + vec_oprnds.reserve_exact (nargs); + vect_get_slp_defs (vinfo, slp_node, &vec_oprnds); for (j = 0; j < ncopies; ++j) { poly_uint64 callee_nelements; @@ -4760,7 +4523,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, { unsigned int k, l, m, o; tree atype; - op = gimple_call_arg (stmt, i + masked_call_offset); + tree op = gimple_call_arg (stmt, i + masked_call_offset); switch (bestn->simdclone->args[i].arg_type) { case SIMD_CLONE_ARG_TYPE_VECTOR: @@ -4780,10 +4543,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, gcc_assert ((k & (k - 1)) == 0); if (m == 0) { - if (!slp_node) - vect_get_vec_defs_for_operand (vinfo, stmt_info, - ncopies * o / k, op, - &vec_oprnds[i]); vec_oprnds_i[i] = 0; vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++]; } @@ -4820,11 +4579,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, { if (m == 0 && l == 0) { - if (!slp_node) - vect_get_vec_defs_for_operand (vinfo, stmt_info, - k * o * ncopies, - op, - &vec_oprnds[i]); vec_oprnds_i[i] = 0; vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++]; } @@ -4888,14 +4642,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, /* The SIMD clone function has the same number of elements as the current function. */ if (m == 0) - { - if (!slp_node) - vect_get_vec_defs_for_operand (vinfo, stmt_info, - o * ncopies, - op, - &vec_oprnds[i]); - vec_oprnds_i[i] = 0; - } + vec_oprnds_i[i] = 0; vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++]; if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) @@ -4904,7 +4651,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, = &LOOP_VINFO_MASKS (loop_vinfo); tree loop_mask = vect_get_loop_mask (loop_vinfo, gsi, - loop_masks, ncopies, + loop_masks, ncopies_in, vectype, j); vec_oprnd0 = prepare_vec_mask (loop_vinfo, @@ -4938,22 +4685,14 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, else if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode)) { atype = bestn->simdclone->args[i].vector_type; - /* Guess the number of lanes represented by atype. */ poly_uint64 atype_subparts = exact_div (bestn->simdclone->simdlen, - num_mask_args); - o = vector_unroll_factor (nunits, atype_subparts); + bestn->simdclone->args[i].linear_step); + o = bestn->simdclone->args[i].linear_step; for (m = j * o; m < (j + 1) * o; m++) { if (m == 0) - { - if (!slp_node) - vect_get_vec_defs_for_operand (vinfo, stmt_info, - o * ncopies, - op, - &vec_oprnds[i]); - vec_oprnds_i[i] = 0; - } + vec_oprnds_i[i] = 0; if (maybe_lt (atype_subparts, TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))) { @@ -4965,10 +4704,25 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, else if (known_eq (atype_subparts, TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))) { + vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++]; + if (loop_vinfo + && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) + { + vec_loop_masks *loop_masks + = &LOOP_VINFO_MASKS (loop_vinfo); + tree loop_mask + = vect_get_loop_mask (loop_vinfo, gsi, + loop_masks, ncopies_in, + vectype, j); + vec_oprnd0 + = prepare_vec_mask (loop_vinfo, + TREE_TYPE (loop_mask), + loop_mask, vec_oprnd0, + gsi); + } /* The vector mask argument matches the input in the number of lanes, but not necessarily in the mode. */ - vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++]; tree st = lang_hooks.types.type_for_mode (TYPE_MODE (TREE_TYPE (vec_oprnd0)), 1); vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, st, @@ -5088,13 +4842,20 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, gcc_assert (bestn->simdclone->args[mask_i].arg_type == SIMD_CLONE_ARG_TYPE_MASK); - tree masktype = bestn->simdclone->args[mask_i].vector_type; + tree mask_argtype = bestn->simdclone->args[mask_i].vector_type; + tree mask_vectype; if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode)) - /* Guess the number of lanes represented by masktype. */ - callee_nelements = exact_div (bestn->simdclone->simdlen, - bestn->simdclone->nargs - nargs); + { + callee_nelements = exact_div (bestn->simdclone->simdlen, + bestn->simdclone->args[i].linear_step); + mask_vectype = get_related_vectype_for_scalar_type + (vinfo->vector_mode, TREE_TYPE (vectype), callee_nelements); + } else - callee_nelements = TYPE_VECTOR_SUBPARTS (masktype); + { + mask_vectype = mask_argtype; + callee_nelements = TYPE_VECTOR_SUBPARTS (mask_vectype); + } o = vector_unroll_factor (nunits, callee_nelements); for (m = j * o; m < (j + 1) * o; m++) { @@ -5102,10 +4863,11 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, { vec_loop_masks *loop_masks = &LOOP_VINFO_MASKS (loop_vinfo); mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks, - ncopies, masktype, j); + ncopies * o, mask_vectype, m); } else - mask = vect_build_all_ones_mask (vinfo, stmt_info, masktype); + mask = vect_build_all_ones_mask (vinfo, stmt_info, + mask_argtype); gassign *new_stmt; if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode)) @@ -5123,23 +4885,18 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, mask); gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT); /* Then zero-extend to the mask mode. */ - mask = fold_build1 (NOP_EXPR, masktype, + mask = fold_build1 (NOP_EXPR, mask_argtype, gimple_get_lhs (new_stmt)); } else if (bestn->simdclone->mask_mode == VOIDmode) - { - tree one = fold_convert (TREE_TYPE (masktype), - integer_one_node); - tree zero = fold_convert (TREE_TYPE (masktype), - integer_zero_node); - mask = build3 (VEC_COND_EXPR, masktype, mask, - build_vector_from_val (masktype, one), - build_vector_from_val (masktype, zero)); - } + mask = build3 (VEC_COND_EXPR, mask_argtype, mask, + build_one_cst (mask_argtype), + build_zero_cst (mask_argtype)); else gcc_unreachable (); - new_stmt = gimple_build_assign (make_ssa_name (masktype), mask); + new_stmt = gimple_build_assign (make_ssa_name (mask_argtype), + mask); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); mask = gimple_assign_lhs (new_stmt); @@ -5188,13 +4945,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, new_stmt = gimple_build_assign (make_ssa_name (vectype), t); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - if (j == 0 && l == 0) - *vec_stmt = new_stmt; - if (slp_node) - SLP_TREE_VEC_DEFS (slp_node) - .quick_push (gimple_assign_lhs (new_stmt)); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + SLP_TREE_VEC_DEFS (slp_node) + .quick_push (gimple_assign_lhs (new_stmt)); } if (ratype) @@ -5237,13 +4989,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - if ((unsigned) j == k - 1) - *vec_stmt = new_stmt; - if (slp_node) - SLP_TREE_VEC_DEFS (slp_node) - .quick_push (gimple_assign_lhs (new_stmt)); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + SLP_TREE_VEC_DEFS (slp_node) + .quick_push (gimple_assign_lhs (new_stmt)); continue; } else if (ratype) @@ -5264,12 +5011,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, } } - if (j == 0) - *vec_stmt = new_stmt; - if (slp_node) + if (gimple_get_lhs (new_stmt)) SLP_TREE_VEC_DEFS (slp_node).quick_push (gimple_get_lhs (new_stmt)); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } for (i = 0; i < nargs; ++i) @@ -5282,26 +5025,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, /* Mark the clone as no longer being a candidate for GC. */ bestn->gc_candidate = false; - /* The call in STMT might prevent it from being removed in dce. - We however cannot remove it here, due to the way the ssa name - it defines is mapped to the new definition. So just replace - rhs of the statement with something harmless. */ - - if (slp_node) - return true; - - gimple *new_stmt; - if (scalar_dest) - { - type = TREE_TYPE (scalar_dest); - lhs = gimple_call_lhs (vect_orig_stmt (stmt_info)->stmt); - new_stmt = gimple_build_assign (lhs, build_zero_cst (type)); - } - else - new_stmt = gimple_build_nop (); - vinfo->replace_stmt (gsi, vect_orig_stmt (stmt_info), new_stmt); - unlink_stmt_vdef (stmt); - return true; } @@ -5372,12 +5095,8 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds, else { /* This is the last step of the conversion sequence. Store the - vectors in SLP_NODE or in vector info of the scalar statement - (or in STMT_VINFO_RELATED_STMT chain). */ - if (slp_node) - slp_node->push_vec_def (new_stmt); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + vectors in SLP_NODE. */ + slp_node->push_vec_def (new_stmt); } } @@ -5515,30 +5234,29 @@ vect_create_half_widening_stmts (vec_info *vinfo, /* Check if STMT_INFO performs a conversion operation that can be vectorized. - If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized - stmt to replace it, put it in VEC_STMT, and insert it at GSI. + If COST_VEC is passed, calculate costs but don't change anything, + otherwise, vectorize STMT_INFO: create a vectorized stmt to replace + it, and insert it at GSI. Return true if STMT_INFO is vectorizable in this way. */ static bool vectorizable_conversion (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - gimple **vec_stmt, slp_tree slp_node, + slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree vec_dest, cvt_op = NULL_TREE; tree scalar_dest; tree op0, op1 = NULL_TREE; - loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); tree_code tc1; code_helper code, code1, code2; code_helper codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK; tree new_temp; enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type}; - int ndts = 2; poly_uint64 nunits_in; poly_uint64 nunits_out; tree vectype_out, vectype_in; - int ncopies, i; + int i; tree lhs_type, rhs_type; /* For conversions between floating point and integer, there're 2 NARROW cases. NARROW_SRC is for FLOAT_EXPR, means @@ -5553,6 +5271,7 @@ vectorizable_conversion (vec_info *vinfo, vec<tree> vec_oprnds1 = vNULL; tree vop0; bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo); + loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); int multi_step_cvt = 0; vec<tree> interm_types = vNULL; tree intermediate_type, cvt_type = NULL_TREE; @@ -5565,7 +5284,7 @@ vectorizable_conversion (vec_info *vinfo, return false; if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def - && ! vec_stmt) + && cost_vec) return false; gimple* stmt = stmt_info->stmt; @@ -5605,11 +5324,11 @@ vectorizable_conversion (vec_info *vinfo, /* Check types of lhs and rhs. */ scalar_dest = gimple_get_lhs (stmt); lhs_type = TREE_TYPE (scalar_dest); - vectype_out = STMT_VINFO_VECTYPE (stmt_info); + vectype_out = SLP_TREE_VECTYPE (slp_node); /* Check the operands of the operation. */ slp_tree slp_op0, slp_op1 = NULL; - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, + if (!vect_is_simple_use (vinfo, slp_node, 0, &op0, &slp_op0, &dt[0], &vectype_in)) { if (dump_enabled_p ()) @@ -5627,15 +5346,12 @@ vectorizable_conversion (vec_info *vinfo, return false; if (!VECTOR_BOOLEAN_TYPE_P (vectype_out) - && ((INTEGRAL_TYPE_P (lhs_type) - && !type_has_mode_precision_p (lhs_type)) - || (INTEGRAL_TYPE_P (rhs_type) - && !type_has_mode_precision_p (rhs_type)))) + && INTEGRAL_TYPE_P (lhs_type) + && !type_has_mode_precision_p (lhs_type)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "type conversion to/from bit-precision unsupported." - "\n"); + "type conversion to bit-precision unsupported\n"); return false; } @@ -5648,7 +5364,7 @@ vectorizable_conversion (vec_info *vinfo, op1 = is_gimple_assign (stmt) ? gimple_assign_rhs2 (stmt) : gimple_call_arg (stmt, 0); tree vectype1_in; - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1, + if (!vect_is_simple_use (vinfo, slp_node, 1, &op1, &slp_op1, &dt[1], &vectype1_in)) { if (dump_enabled_p ()) @@ -5666,7 +5382,7 @@ vectorizable_conversion (vec_info *vinfo, from the scalar type. */ if (!vectype_in) vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node); - if (vec_stmt) + if (!cost_vec) gcc_assert (vectype_in); if (!vectype_in) { @@ -5678,7 +5394,7 @@ vectorizable_conversion (vec_info *vinfo, } if (VECTOR_BOOLEAN_TYPE_P (vectype_out) - && !VECTOR_BOOLEAN_TYPE_P (vectype_in)) + != VECTOR_BOOLEAN_TYPE_P (vectype_in)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -5703,25 +5419,12 @@ vectorizable_conversion (vec_info *vinfo, modifier = WIDEN; } - /* Multiple types in SLP are handled by creating the appropriate number of - vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in - case of SLP. */ - if (slp_node) - ncopies = 1; - else if (modifier == NARROW_DST) - ncopies = vect_get_num_copies (loop_vinfo, vectype_out); - else - ncopies = vect_get_num_copies (loop_vinfo, vectype_in); - - /* Sanity check: make sure that at least one copy of the vectorized stmt - needs to be generated. */ - gcc_assert (ncopies >= 1); - bool found_mode = false; scalar_mode lhs_mode = SCALAR_TYPE_MODE (lhs_type); scalar_mode rhs_mode = SCALAR_TYPE_MODE (rhs_type); opt_scalar_mode rhs_mode_iter; - auto_vec<std::pair<tree, tree_code> > converts; + auto_vec<std::pair<tree, tree_code>, 2> converts; + bool evenodd_ok = false; /* Supportable by target? */ switch (modifier) @@ -5769,10 +5472,28 @@ vectorizable_conversion (vec_info *vinfo, gcc_assert (!(multi_step_cvt && op_type == binary_op)); break; } - if (supportable_widening_operation (vinfo, code, stmt_info, - vectype_out, vectype_in, &code1, - &code2, &multi_step_cvt, - &interm_types)) + /* Elements in a vector can only be reordered if used in a reduction + operation only. */ + if (code == WIDEN_MULT_EXPR + && loop_vinfo + && !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info) + /* For a SLP reduction we cannot swizzle lanes, detecting a + reduction chain isn't possible here. */ + && SLP_TREE_LANES (slp_node) == 1) + { + /* ??? There is no way to look for SLP uses, so work on + the stmt and what the stmt-based cycle detection gives us. */ + tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt); + stmt_vec_info use_stmt_info + = lhs ? loop_vinfo->lookup_single_use (lhs) : NULL; + if (use_stmt_info + && STMT_VINFO_REDUC_DEF (use_stmt_info)) + evenodd_ok = true; + } + if (supportable_widening_operation (code, vectype_out, vectype_in, + evenodd_ok, &code1, + &code2, &multi_step_cvt, + &interm_types)) { /* Binary widening operation can only be supported directly by the architecture. */ @@ -5806,18 +5527,17 @@ vectorizable_conversion (vec_info *vinfo, goto unsupported; codecvt1 = tc1; } - else if (!supportable_widening_operation (vinfo, code, - stmt_info, vectype_out, - cvt_type, &codecvt1, + else if (!supportable_widening_operation (code, vectype_out, + cvt_type, evenodd_ok, + &codecvt1, &codecvt2, &multi_step_cvt, &interm_types)) continue; else gcc_assert (multi_step_cvt == 0); - if (supportable_widening_operation (vinfo, NOP_EXPR, stmt_info, - cvt_type, - vectype_in, &code1, + if (supportable_widening_operation (NOP_EXPR, cvt_type, + vectype_in, evenodd_ok, &code1, &code2, &multi_step_cvt, &interm_types)) { @@ -5870,28 +5590,26 @@ vectorizable_conversion (vec_info *vinfo, truncate it to cvt_type and the do FLOAT_EXPR. */ else if (code == FLOAT_EXPR) { - wide_int op_min_value, op_max_value; - if (slp_node) + if (cost_vec) { + wide_int op_min_value, op_max_value; tree def; + /* ??? Merge ranges in case of more than one lane. */ if (SLP_TREE_LANES (slp_op0) != 1 || !(def = vect_get_slp_scalar_def (slp_op0, 0)) || !vect_get_range_info (def, &op_min_value, &op_max_value)) goto unsupported; + + if ((wi::min_precision (op_max_value, SIGNED) + > GET_MODE_BITSIZE (lhs_mode)) + || (wi::min_precision (op_min_value, SIGNED) + > GET_MODE_BITSIZE (lhs_mode))) + goto unsupported; } - else if (!vect_get_range_info (op0, &op_min_value, &op_max_value)) - goto unsupported; cvt_type = build_nonstandard_integer_type (GET_MODE_BITSIZE (lhs_mode), 0); - if (cvt_type == NULL_TREE - || (wi::min_precision (op_max_value, SIGNED) - > TYPE_PRECISION (cvt_type)) - || (wi::min_precision (op_min_value, SIGNED) - > TYPE_PRECISION (cvt_type))) - goto unsupported; - cvt_type = get_same_sized_vectype (cvt_type, vectype_out); if (cvt_type == NULL_TREE) goto unsupported; @@ -5914,11 +5632,24 @@ vectorizable_conversion (vec_info *vinfo, gcc_unreachable (); } - if (!vec_stmt) /* transformation not required. */ + if (modifier == WIDEN + && loop_vinfo + && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) + && (code1 == VEC_WIDEN_MULT_EVEN_EXPR + || widening_evenodd_fn_p (code1))) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "can't use a fully-masked loop because" + " widening operation on even/odd elements" + " mixes up lanes.\n"); + LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false; + } + + if (cost_vec) /* transformation not required. */ { - if (slp_node - && (!vect_maybe_update_slp_op_vectype (slp_op0, vectype_in) - || !vect_maybe_update_slp_op_vectype (slp_op1, vectype_in))) + if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype_in) + || !vect_maybe_update_slp_op_vectype (slp_op1, vectype_in)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -5926,34 +5657,29 @@ vectorizable_conversion (vec_info *vinfo, return false; } DUMP_VECT_SCOPE ("vectorizable_conversion"); + unsigned int nvectors = vect_get_num_copies (vinfo, slp_node); if (modifier == NONE) { - STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type; - vect_model_simple_cost (vinfo, stmt_info, - ncopies * (1 + multi_step_cvt), - dt, ndts, slp_node, cost_vec); + SLP_TREE_TYPE (slp_node) = type_conversion_vec_info_type; + vect_model_simple_cost (vinfo, (1 + multi_step_cvt), + slp_node, cost_vec); } else if (modifier == NARROW_SRC || modifier == NARROW_DST) { - STMT_VINFO_TYPE (stmt_info) = type_demotion_vec_info_type; + SLP_TREE_TYPE (slp_node) = type_demotion_vec_info_type; /* The final packing step produces one vector result per copy. */ - unsigned int nvectors - = (slp_node ? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) : ncopies); - vect_model_promotion_demotion_cost (stmt_info, dt, nvectors, + vect_model_promotion_demotion_cost (slp_node, nvectors, multi_step_cvt, cost_vec, widen_arith); } else { - STMT_VINFO_TYPE (stmt_info) = type_promotion_vec_info_type; + SLP_TREE_TYPE (slp_node) = type_promotion_vec_info_type; /* The initial unpacking step produces two vector results per copy. MULTI_STEP_CVT is 0 for a single conversion, so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */ - unsigned int nvectors - = (slp_node - ? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) >> multi_step_cvt - : ncopies * 2); - vect_model_promotion_demotion_cost (stmt_info, dt, nvectors, + vect_model_promotion_demotion_cost (slp_node, + nvectors >> multi_step_cvt, multi_step_cvt, cost_vec, widen_arith); } @@ -5963,8 +5689,7 @@ vectorizable_conversion (vec_info *vinfo, /* Transform. */ if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "transform conversion. ncopies = %d.\n", ncopies); + dump_printf_loc (MSG_NOTE, vect_location, "transform conversion.\n"); if (op_type == binary_op) { @@ -6003,24 +5728,10 @@ vectorizable_conversion (vec_info *vinfo, widen_or_narrow_float_p ? vectype_out : cvt_type); - int ninputs = 1; - if (!slp_node) - { - if (modifier == WIDEN) - ; - else if (modifier == NARROW_SRC || modifier == NARROW_DST) - { - if (multi_step_cvt) - ninputs = vect_pow2 (multi_step_cvt); - ninputs *= 2; - } - } - switch (modifier) { case NONE: - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, - op0, vectype_in, &vec_oprnds0); + vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0); /* vec_dest is intermediate type operand when multi_step_cvt. */ if (multi_step_cvt) { @@ -6046,10 +5757,7 @@ vectorizable_conversion (vec_info *vinfo, gimple_set_lhs (new_stmt, new_temp); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - if (slp_node) - slp_node->push_vec_def (new_stmt); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + slp_node->push_vec_def (new_stmt); } break; @@ -6058,10 +5766,9 @@ vectorizable_conversion (vec_info *vinfo, of elements that we can fit in a vectype (nunits), we have to generate more than one vector stmt - i.e - we need to "unroll" the vector stmt by a factor VF/nunits. */ - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies * ninputs, - op0, vectype_in, &vec_oprnds0, + vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0, code == WIDEN_LSHIFT_EXPR ? NULL_TREE : op1, - vectype_in, &vec_oprnds1); + &vec_oprnds1); if (code == WIDEN_LSHIFT_EXPR) { int oprnds_size = vec_oprnds0.length (); @@ -6102,10 +5809,7 @@ vectorizable_conversion (vec_info *vinfo, else new_stmt = SSA_NAME_DEF_STMT (vop0); - if (slp_node) - slp_node->push_vec_def (new_stmt); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + slp_node->push_vec_def (new_stmt); } break; @@ -6115,8 +5819,7 @@ vectorizable_conversion (vec_info *vinfo, of elements that we can fit in a vectype (nunits), we have to generate more than one vector stmt - i.e - we need to "unroll" the vector stmt by a factor VF/nunits. */ - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies * ninputs, - op0, vectype_in, &vec_oprnds0); + vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0); /* Arguments are ready. Create the new vector stmts. */ if (cvt_type && modifier == NARROW_DST) FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0) @@ -6148,16 +5851,11 @@ vectorizable_conversion (vec_info *vinfo, /* This is the last step of the conversion sequence. Store the vectors in SLP_NODE or in vector info of the scalar statement (or in STMT_VINFO_RELATED_STMT chain). */ - if (slp_node) - slp_node->push_vec_def (new_stmt); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + slp_node->push_vec_def (new_stmt); } } break; } - if (!slp_node) - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; vec_oprnds0.release (); vec_oprnds1.release (); @@ -6193,24 +5891,22 @@ vect_nop_conversion_p (stmt_vec_info stmt_info) /* Function vectorizable_assignment. Check if STMT_INFO performs an assignment (copy) that can be vectorized. - If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized - stmt to replace it, put it in VEC_STMT, and insert it at GSI. + If COST_VEC is passed, calculate costs but don't change anything, + otherwise, vectorize STMT_INFO: create a vectorized stmt to replace + it, and insert it at GSI. Return true if STMT_INFO is vectorizable in this way. */ static bool vectorizable_assignment (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - gimple **vec_stmt, slp_tree slp_node, + slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree vec_dest; tree scalar_dest; tree op; - loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); tree new_temp; enum vect_def_type dt[1] = {vect_unknown_def_type}; - int ndts = 1; - int ncopies; int i; vec<tree> vec_oprnds = vNULL; tree vop; @@ -6222,7 +5918,7 @@ vectorizable_assignment (vec_info *vinfo, return false; if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def - && ! vec_stmt) + && cost_vec) return false; /* Is vectorizable assignment? */ @@ -6243,21 +5939,11 @@ vectorizable_assignment (vec_info *vinfo, || CONVERT_EXPR_CODE_P (code))) return false; - tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vectype = SLP_TREE_VECTYPE (slp_node); poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); - /* Multiple types in SLP are handled by creating the appropriate number of - vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in - case of SLP. */ - if (slp_node) - ncopies = 1; - else - ncopies = vect_get_num_copies (loop_vinfo, vectype); - - gcc_assert (ncopies >= 1); - slp_tree slp_op; - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 0, &op, &slp_op, + if (!vect_is_simple_use (vinfo, slp_node, 0, &op, &slp_op, &dt[0], &vectype_in)) { if (dump_enabled_p ()) @@ -6314,21 +6000,19 @@ vectorizable_assignment (vec_info *vinfo, return false; } - if (!vec_stmt) /* transformation not required. */ + if (cost_vec) /* transformation not required. */ { - if (slp_node - && !vect_maybe_update_slp_op_vectype (slp_op, vectype_in)) + if (!vect_maybe_update_slp_op_vectype (slp_op, vectype_in)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, "incompatible vector types for invariants\n"); return false; } - STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type; + SLP_TREE_TYPE (slp_node) = assignment_vec_info_type; DUMP_VECT_SCOPE ("vectorizable_assignment"); if (!vect_nop_conversion_p (stmt_info)) - vect_model_simple_cost (vinfo, stmt_info, ncopies, dt, ndts, slp_node, - cost_vec); + vect_model_simple_cost (vinfo, 1, slp_node, cost_vec); return true; } @@ -6340,7 +6024,7 @@ vectorizable_assignment (vec_info *vinfo, vec_dest = vect_create_destination_var (scalar_dest, vectype); /* Handle use. */ - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, op, &vec_oprnds); + vect_get_vec_defs (vinfo, slp_node, op, &vec_oprnds); /* Arguments are ready. create the new vector stmt. */ FOR_EACH_VEC_ELT (vec_oprnds, i, vop) @@ -6352,13 +6036,8 @@ vectorizable_assignment (vec_info *vinfo, new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - if (slp_node) - slp_node->push_vec_def (new_stmt); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + slp_node->push_vec_def (new_stmt); } - if (!slp_node) - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; vec_oprnds.release (); return true; @@ -6393,14 +6072,15 @@ vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type) /* Function vectorizable_shift. Check if STMT_INFO performs a shift operation that can be vectorized. - If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized - stmt to replace it, put it in VEC_STMT, and insert it at GSI. + If COST_VEC is passed, calculate costs but don't change anything, + otherwise, vectorize STMT_INFO: create a vectorized stmt to replace + it, and insert it at GSI. Return true if STMT_INFO is vectorizable in this way. */ static bool vectorizable_shift (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - gimple **vec_stmt, slp_tree slp_node, + slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree vec_dest; @@ -6408,7 +6088,6 @@ vectorizable_shift (vec_info *vinfo, tree op0, op1 = NULL; tree vec_oprnd1 = NULL_TREE; tree vectype; - loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); enum tree_code code; machine_mode vec_mode; tree new_temp; @@ -6416,12 +6095,10 @@ vectorizable_shift (vec_info *vinfo, int icode; machine_mode optab_op2_mode; enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type}; - int ndts = 2; poly_uint64 nunits_in; poly_uint64 nunits_out; tree vectype_out; tree op1_vectype; - int ncopies; int i; vec<tree> vec_oprnds0 = vNULL; vec<tree> vec_oprnds1 = vNULL; @@ -6436,7 +6113,7 @@ vectorizable_shift (vec_info *vinfo, if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle - && ! vec_stmt) + && cost_vec) return false; /* Is STMT a vectorizable binary/unary operation? */ @@ -6454,7 +6131,7 @@ vectorizable_shift (vec_info *vinfo, return false; scalar_dest = gimple_assign_lhs (stmt); - vectype_out = STMT_VINFO_VECTYPE (stmt_info); + vectype_out = SLP_TREE_VECTYPE (slp_node); if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest))) { if (dump_enabled_p ()) @@ -6464,7 +6141,7 @@ vectorizable_shift (vec_info *vinfo, } slp_tree slp_op0; - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, + if (!vect_is_simple_use (vinfo, slp_node, 0, &op0, &slp_op0, &dt[0], &vectype)) { if (dump_enabled_p ()) @@ -6476,7 +6153,7 @@ vectorizable_shift (vec_info *vinfo, from the scalar type. */ if (!vectype) vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0), slp_node); - if (vec_stmt) + if (!cost_vec) gcc_assert (vectype); if (!vectype) { @@ -6493,7 +6170,7 @@ vectorizable_shift (vec_info *vinfo, stmt_vec_info op1_def_stmt_info; slp_tree slp_op1; - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1, &op1, &slp_op1, + if (!vect_is_simple_use (vinfo, slp_node, 1, &op1, &slp_op1, &dt[1], &op1_vectype, &op1_def_stmt_info)) { if (dump_enabled_p ()) @@ -6502,23 +6179,13 @@ vectorizable_shift (vec_info *vinfo, return false; } - /* Multiple types in SLP are handled by creating the appropriate number of - vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in - case of SLP. */ - if (slp_node) - ncopies = 1; - else - ncopies = vect_get_num_copies (loop_vinfo, vectype); - - gcc_assert (ncopies >= 1); - /* Determine whether the shift amount is a vector, or scalar. If the shift/rotate amount is a vector, use the vector/vector shift optabs. */ if ((dt[1] == vect_internal_def || dt[1] == vect_induction_def || dt[1] == vect_nested_cycle) - && (!slp_node || SLP_TREE_LANES (slp_node) == 1)) + && SLP_TREE_LANES (slp_node) == 1) scalar_shift_arg = false; else if (dt[1] == vect_constant_def || dt[1] == vect_external_def @@ -6527,29 +6194,26 @@ vectorizable_shift (vec_info *vinfo, /* In SLP, need to check whether the shift count is the same, in loops if it is a constant or invariant, it is always a scalar shift. */ - if (slp_node) - { - vec<stmt_vec_info> stmts = SLP_TREE_SCALAR_STMTS (slp_node); - stmt_vec_info slpstmt_info; + vec<stmt_vec_info> stmts = SLP_TREE_SCALAR_STMTS (slp_node); + stmt_vec_info slpstmt_info; - FOR_EACH_VEC_ELT (stmts, k, slpstmt_info) - if (slpstmt_info) - { - gassign *slpstmt = as_a <gassign *> (slpstmt_info->stmt); - if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0)) - scalar_shift_arg = false; - } + FOR_EACH_VEC_ELT (stmts, k, slpstmt_info) + if (slpstmt_info) + { + gassign *slpstmt = as_a <gassign *> (slpstmt_info->stmt); + if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0)) + scalar_shift_arg = false; + } - /* For internal SLP defs we have to make sure we see scalar stmts - for all vector elements. - ??? For different vectors we could resort to a different - scalar shift operand but code-generation below simply always - takes the first. */ - if (dt[1] == vect_internal_def - && maybe_ne (nunits_out * SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node), - stmts.length ())) - scalar_shift_arg = false; - } + /* For internal SLP defs we have to make sure we see scalar stmts + for all vector elements. + ??? For different vectors we could resort to a different + scalar shift operand but code-generation below simply always + takes the first. */ + if (dt[1] == vect_internal_def + && maybe_ne (nunits_out * vect_get_num_copies (vinfo, slp_node), + stmts.length ())) + scalar_shift_arg = false; /* If the shift amount is computed by a pattern stmt we cannot use the scalar amount directly thus give up and use a vector @@ -6583,8 +6247,7 @@ vectorizable_shift (vec_info *vinfo, TYPE_VECTOR_SUBPARTS (vectype)) || TYPE_MODE (op1_vectype) != TYPE_MODE (vectype)); if (incompatible_op1_vectype_p - && (!slp_node - || SLP_TREE_DEF_TYPE (slp_op1) != vect_constant_def + && (SLP_TREE_DEF_TYPE (slp_op1) != vect_constant_def || slp_op1->refcnt != 1)) { if (dump_enabled_p ()) @@ -6665,16 +6328,15 @@ vectorizable_shift (vec_info *vinfo, if (vect_emulated_vector_p (vectype)) return false; - if (!vec_stmt) /* transformation not required. */ + if (cost_vec) /* transformation not required. */ { - if (slp_node - && (!vect_maybe_update_slp_op_vectype (slp_op0, vectype) - || ((!scalar_shift_arg || dt[1] == vect_internal_def) - && (!incompatible_op1_vectype_p - || dt[1] == vect_constant_def) - && !vect_maybe_update_slp_op_vectype - (slp_op1, - incompatible_op1_vectype_p ? vectype : op1_vectype)))) + if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype) + || ((!scalar_shift_arg || dt[1] == vect_internal_def) + && (!incompatible_op1_vectype_p + || dt[1] == vect_constant_def) + && !vect_maybe_update_slp_op_vectype + (slp_op1, + incompatible_op1_vectype_p ? vectype : op1_vectype))) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -6682,24 +6344,20 @@ vectorizable_shift (vec_info *vinfo, return false; } /* Now adjust the constant shift amount in place. */ - if (slp_node - && incompatible_op1_vectype_p + if (incompatible_op1_vectype_p && dt[1] == vect_constant_def) - { - for (unsigned i = 0; - i < SLP_TREE_SCALAR_OPS (slp_op1).length (); ++i) - { - SLP_TREE_SCALAR_OPS (slp_op1)[i] - = fold_convert (TREE_TYPE (vectype), - SLP_TREE_SCALAR_OPS (slp_op1)[i]); - gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i]) - == INTEGER_CST)); - } - } - STMT_VINFO_TYPE (stmt_info) = shift_vec_info_type; + for (unsigned i = 0; + i < SLP_TREE_SCALAR_OPS (slp_op1).length (); ++i) + { + SLP_TREE_SCALAR_OPS (slp_op1)[i] + = fold_convert (TREE_TYPE (vectype), + SLP_TREE_SCALAR_OPS (slp_op1)[i]); + gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i]) + == INTEGER_CST)); + } + SLP_TREE_TYPE (slp_node) = shift_vec_info_type; DUMP_VECT_SCOPE ("vectorizable_shift"); - vect_model_simple_cost (vinfo, stmt_info, ncopies, dt, - scalar_shift_arg ? 1 : ndts, slp_node, cost_vec); + vect_model_simple_cost (vinfo, 1, slp_node, cost_vec); return true; } @@ -6709,18 +6367,10 @@ vectorizable_shift (vec_info *vinfo, dump_printf_loc (MSG_NOTE, vect_location, "transform binary/unary operation.\n"); - if (incompatible_op1_vectype_p && !slp_node) - { - gcc_assert (!scalar_shift_arg && was_scalar_shift_arg); - op1 = fold_convert (TREE_TYPE (vectype), op1); - if (dt[1] != vect_constant_def) - op1 = vect_init_vector (vinfo, stmt_info, op1, - TREE_TYPE (vectype), NULL); - } - /* Handle def. */ vec_dest = vect_create_destination_var (scalar_dest, vectype); + unsigned nvectors = vect_get_num_copies (vinfo, slp_node); if (scalar_shift_arg && dt[1] != vect_internal_def) { /* Vector shl and shr insn patterns can be defined with scalar @@ -6734,19 +6384,18 @@ vectorizable_shift (vec_info *vinfo, dump_printf_loc (MSG_NOTE, vect_location, "operand 1 using scalar mode.\n"); vec_oprnd1 = op1; - vec_oprnds1.create (slp_node ? slp_node->vec_stmts_size : ncopies); + vec_oprnds1.create (nvectors); vec_oprnds1.quick_push (vec_oprnd1); - /* Store vec_oprnd1 for every vector stmt to be created. - We check during the analysis that all the shift arguments - are the same. - TODO: Allow different constants for different vector - stmts generated for an SLP instance. */ - for (k = 0; - k < (slp_node ? slp_node->vec_stmts_size - 1 : ncopies - 1); k++) + /* Store vec_oprnd1 for every vector stmt to be created. + We check during the analysis that all the shift arguments + are the same. + TODO: Allow different constants for different vector + stmts generated for an SLP instance. */ + for (k = 0; k < nvectors - 1; k++) vec_oprnds1.quick_push (vec_oprnd1); } } - else if (!scalar_shift_arg && slp_node && incompatible_op1_vectype_p) + else if (!scalar_shift_arg && incompatible_op1_vectype_p) { if (was_scalar_shift_arg) { @@ -6760,8 +6409,8 @@ vectorizable_shift (vec_info *vinfo, gsi); vec_oprnd1 = vect_init_vector (vinfo, stmt_info, op1, vectype, gsi); - vec_oprnds1.create (slp_node->vec_stmts_size); - for (k = 0; k < slp_node->vec_stmts_size; k++) + vec_oprnds1.create (nvectors); + for (k = 0; k < nvectors; k++) vec_oprnds1.quick_push (vec_oprnd1); } else if (dt[1] == vect_constant_def) @@ -6774,7 +6423,7 @@ vectorizable_shift (vec_info *vinfo, /* vec_oprnd1 is available if operand 1 should be of a scalar-type (a special case for certain kind of vector shifts); otherwise, operand 1 should be of a vector type (the usual case). */ - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, + vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0, vec_oprnd1 ? NULL_TREE : op1, &vec_oprnds1); @@ -6802,15 +6451,9 @@ vectorizable_shift (vec_info *vinfo, new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - if (slp_node) - slp_node->push_vec_def (new_stmt); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + slp_node->push_vec_def (new_stmt); } - if (!slp_node) - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - vec_oprnds0.release (); vec_oprnds1.release (); @@ -6821,14 +6464,15 @@ vectorizable_shift (vec_info *vinfo, Check if STMT_INFO performs a binary, unary or ternary operation that can be vectorized. - If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized - stmt to replace it, put it in VEC_STMT, and insert it at GSI. + If COST_VEC is passed, calculate costs but don't change anything, + otherwise, vectorize STMT_INFO: create a vectorized stmt to replace + it, and insert it at GSI. Return true if STMT_INFO is vectorizable in this way. */ static bool vectorizable_operation (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - gimple **vec_stmt, slp_tree slp_node, + slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree vec_dest; @@ -6844,12 +6488,9 @@ vectorizable_operation (vec_info *vinfo, bool target_support_p; enum vect_def_type dt[3] = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type}; - int ndts = 3; poly_uint64 nunits_in; poly_uint64 nunits_out; tree vectype_out; - unsigned int ncopies; - int vec_num; int i; vec<tree> vec_oprnds0 = vNULL; vec<tree> vec_oprnds1 = vNULL; @@ -6861,7 +6502,7 @@ vectorizable_operation (vec_info *vinfo, return false; if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def - && ! vec_stmt) + && cost_vec) return false; /* Is STMT a vectorizable binary/unary operation? */ @@ -6909,7 +6550,7 @@ vectorizable_operation (vec_info *vinfo, } scalar_dest = gimple_assign_lhs (stmt); - vectype_out = STMT_VINFO_VECTYPE (stmt_info); + vectype_out = SLP_TREE_VECTYPE (slp_node); /* Most operations cannot handle bit-precision types without extra truncations. */ @@ -6928,7 +6569,7 @@ vectorizable_operation (vec_info *vinfo, } slp_tree slp_op0; - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, + if (!vect_is_simple_use (vinfo, slp_node, 0, &op0, &slp_op0, &dt[0], &vectype)) { if (dump_enabled_p ()) @@ -6962,7 +6603,7 @@ vectorizable_operation (vec_info *vinfo, vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0), slp_node); } - if (vec_stmt) + if (!cost_vec) gcc_assert (vectype); if (!vectype) { @@ -6984,7 +6625,7 @@ vectorizable_operation (vec_info *vinfo, slp_tree slp_op1 = NULL, slp_op2 = NULL; if (op_type == binary_op || op_type == ternary_op) { - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, + if (!vect_is_simple_use (vinfo, slp_node, 1, &op1, &slp_op1, &dt[1], &vectype2)) { if (dump_enabled_p ()) @@ -7002,7 +6643,7 @@ vectorizable_operation (vec_info *vinfo, } if (op_type == ternary_op) { - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, + if (!vect_is_simple_use (vinfo, slp_node, 2, &op2, &slp_op2, &dt[2], &vectype3)) { if (dump_enabled_p ()) @@ -7020,20 +6661,8 @@ vectorizable_operation (vec_info *vinfo, } /* Multiple types in SLP are handled by creating the appropriate number of - vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in - case of SLP. */ - if (slp_node) - { - ncopies = 1; - vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); - } - else - { - ncopies = vect_get_num_copies (loop_vinfo, vectype); - vec_num = 1; - } - - gcc_assert (ncopies >= 1); + vectorized stmts for each SLP node. */ + auto vec_num = vect_get_num_copies (vinfo, slp_node); /* Reject attempts to combine mask types with nonmask types, e.g. if we have an AND between a (nonmask) boolean loaded from memory and @@ -7075,11 +6704,12 @@ vectorizable_operation (vec_info *vinfo, ops we have to lower the lowering code assumes we are dealing with word_mode. */ if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype)) + || !GET_MODE_SIZE (vec_mode).is_constant () || (((code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR) - || !target_support_p) - && maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD)) + || !target_support_p) + && maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD)) /* Check only during analysis. */ - || (!vec_stmt && !vect_can_vectorize_without_simd_p (code))) + || (cost_vec && !vect_can_vectorize_without_simd_p (code))) { if (dump_enabled_p ()) dump_printf (MSG_NOTE, "using word mode not possible.\n"); @@ -7091,7 +6721,7 @@ vectorizable_operation (vec_info *vinfo, using_emulated_vectors_p = true; } - int reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info); + int reduc_idx = SLP_TREE_REDUC_IDX (slp_node); vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL); vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL); internal_fn cond_fn = get_conditional_internal_fn (code); @@ -7108,7 +6738,7 @@ vectorizable_operation (vec_info *vinfo, bool mask_out_inactive = ((!is_invariant && gimple_could_trap_p (stmt)) || reduc_idx >= 0); - if (!vec_stmt) /* transformation not required. */ + if (cost_vec) /* transformation not required. */ { if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) @@ -7117,12 +6747,12 @@ vectorizable_operation (vec_info *vinfo, if (cond_len_fn != IFN_LAST && direct_internal_fn_supported_p (cond_len_fn, vectype, OPTIMIZE_FOR_SPEED)) - vect_record_loop_len (loop_vinfo, lens, ncopies * vec_num, vectype, + vect_record_loop_len (loop_vinfo, lens, vec_num, vectype, 1); else if (cond_fn != IFN_LAST && direct_internal_fn_supported_p (cond_fn, vectype, OPTIMIZE_FOR_SPEED)) - vect_record_loop_mask (loop_vinfo, masks, ncopies * vec_num, + vect_record_loop_mask (loop_vinfo, masks, vec_num, vectype, NULL); else { @@ -7135,10 +6765,9 @@ vectorizable_operation (vec_info *vinfo, } /* Put types on constant and invariant SLP children. */ - if (slp_node - && (!vect_maybe_update_slp_op_vectype (slp_op0, vectype) - || !vect_maybe_update_slp_op_vectype (slp_op1, vectype) - || !vect_maybe_update_slp_op_vectype (slp_op2, vectype))) + if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype) + || !vect_maybe_update_slp_op_vectype (slp_op1, vectype) + || !vect_maybe_update_slp_op_vectype (slp_op2, vectype)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -7146,18 +6775,16 @@ vectorizable_operation (vec_info *vinfo, return false; } - STMT_VINFO_TYPE (stmt_info) = op_vec_info_type; + SLP_TREE_TYPE (slp_node) = op_vec_info_type; DUMP_VECT_SCOPE ("vectorizable_operation"); - vect_model_simple_cost (vinfo, stmt_info, - ncopies, dt, ndts, slp_node, cost_vec); + vect_model_simple_cost (vinfo, 1, slp_node, cost_vec); if (using_emulated_vectors_p) { /* The above vect_model_simple_cost call handles constants in the prologue and (mis-)costs one of the stmts as vector stmt. See below for the actual lowering that will be applied. */ - unsigned n - = slp_node ? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) : ncopies; + unsigned n = vect_get_num_copies (vinfo, slp_node); switch (code) { case PLUS_EXPR: @@ -7206,64 +6833,25 @@ vectorizable_operation (vec_info *vinfo, vec_dest = vect_create_destination_var (scalar_dest, vectype); vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out); } + /* For reduction operations with undefined overflow behavior make sure to + pun them to unsigned since we change the order of evaluation. + ??? Avoid for in-order reductions? */ + else if (arith_code_with_undefined_signed_overflow (orig_code) + && ANY_INTEGRAL_TYPE_P (vectype) + && TYPE_OVERFLOW_UNDEFINED (vectype) + && SLP_TREE_REDUC_IDX (slp_node) != -1) + { + gcc_assert (orig_code == PLUS_EXPR || orig_code == MINUS_EXPR + || orig_code == MULT_EXPR || orig_code == POINTER_PLUS_EXPR); + vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out); + vectype = unsigned_type_for (vectype); + vec_dest = vect_create_destination_var (scalar_dest, vectype); + } /* Handle def. */ else vec_dest = vect_create_destination_var (scalar_dest, vectype_out); - /* In case the vectorization factor (VF) is bigger than the number - of elements that we can fit in a vectype (nunits), we have to generate - more than one vector stmt - i.e - we need to "unroll" the - vector stmt by a factor VF/nunits. In doing so, we record a pointer - from one copy of the vector stmt to the next, in the field - STMT_VINFO_RELATED_STMT. This is necessary in order to allow following - stages to find the correct vector defs to be used when vectorizing - stmts that use the defs of the current stmt. The example below - illustrates the vectorization process when VF=16 and nunits=4 (i.e., - we need to create 4 vectorized stmts): - - before vectorization: - RELATED_STMT VEC_STMT - S1: x = memref - - - S2: z = x + 1 - - - - step 1: vectorize stmt S1 (done in vectorizable_load. See more details - there): - RELATED_STMT VEC_STMT - VS1_0: vx0 = memref0 VS1_1 - - VS1_1: vx1 = memref1 VS1_2 - - VS1_2: vx2 = memref2 VS1_3 - - VS1_3: vx3 = memref3 - - - S1: x = load - VS1_0 - S2: z = x + 1 - - - - step2: vectorize stmt S2 (done here): - To vectorize stmt S2 we first need to find the relevant vector - def for the first operand 'x'. This is, as usual, obtained from - the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt - that defines 'x' (S1). This way we find the stmt VS1_0, and the - relevant vector def 'vx0'. Having found 'vx0' we can generate - the vector stmt VS2_0, and as usual, record it in the - STMT_VINFO_VEC_STMT of stmt S2. - When creating the second copy (VS2_1), we obtain the relevant vector - def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of - stmt VS1_0. This way we find the stmt VS1_1 and the relevant - vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a - pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0. - Similarly when creating stmts VS2_2 and VS2_3. This is the resulting - chain of stmts and pointers: - RELATED_STMT VEC_STMT - VS1_0: vx0 = memref0 VS1_1 - - VS1_1: vx1 = memref1 VS1_2 - - VS1_2: vx2 = memref2 VS1_3 - - VS1_3: vx3 = memref3 - - - S1: x = load - VS1_0 - VS2_0: vz0 = vx0 + v1 VS2_1 - - VS2_1: vz1 = vx1 + v1 VS2_2 - - VS2_2: vz2 = vx2 + v1 VS2_3 - - VS2_3: vz3 = vx3 + v1 - - - S2: z = x + 1 - VS2_0 */ - - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, + vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0, op1, &vec_oprnds1, op2, &vec_oprnds2); /* Arguments are ready. Create the new vector stmt. */ FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0) @@ -7272,88 +6860,148 @@ vectorizable_operation (vec_info *vinfo, vop1 = ((op_type == binary_op || op_type == ternary_op) ? vec_oprnds1[i] : NULL_TREE); vop2 = ((op_type == ternary_op) ? vec_oprnds2[i] : NULL_TREE); - if (using_emulated_vectors_p - && (code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)) + + if (vec_cvt_dest + && !useless_type_conversion_p (vectype, TREE_TYPE (vop0))) + { + new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop0); + new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR, + new_temp); + new_temp = make_ssa_name (vec_dest, new_stmt); + gimple_assign_set_lhs (new_stmt, new_temp); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); + vop0 = new_temp; + } + if (vop1 + && vec_cvt_dest + && !useless_type_conversion_p (vectype, TREE_TYPE (vop1))) + { + new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop1); + new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR, + new_temp); + new_temp = make_ssa_name (vec_dest, new_stmt); + gimple_assign_set_lhs (new_stmt, new_temp); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); + vop1 = new_temp; + } + if (vop2 + && vec_cvt_dest + && !useless_type_conversion_p (vectype, TREE_TYPE (vop2))) + { + new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop2); + new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR, + new_temp); + new_temp = make_ssa_name (vec_dest, new_stmt); + gimple_assign_set_lhs (new_stmt, new_temp); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); + vop2 = new_temp; + } + + if (using_emulated_vectors_p) { /* Lower the operation. This follows vector lowering. */ - unsigned int width = vector_element_bits (vectype); - tree inner_type = TREE_TYPE (vectype); - tree word_type - = build_nonstandard_integer_type (GET_MODE_BITSIZE (word_mode), 1); - HOST_WIDE_INT max = GET_MODE_MASK (TYPE_MODE (inner_type)); - tree low_bits = build_replicated_int_cst (word_type, width, max >> 1); - tree high_bits - = build_replicated_int_cst (word_type, width, max & ~(max >> 1)); + tree word_type = build_nonstandard_integer_type + (GET_MODE_BITSIZE (vec_mode).to_constant (), 1); tree wvop0 = make_ssa_name (word_type); new_stmt = gimple_build_assign (wvop0, VIEW_CONVERT_EXPR, build1 (VIEW_CONVERT_EXPR, word_type, vop0)); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - tree result_low, signs; - if (code == PLUS_EXPR || code == MINUS_EXPR) + tree wvop1 = NULL_TREE; + if (vop1) { - tree wvop1 = make_ssa_name (word_type); + wvop1 = make_ssa_name (word_type); new_stmt = gimple_build_assign (wvop1, VIEW_CONVERT_EXPR, build1 (VIEW_CONVERT_EXPR, word_type, vop1)); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - signs = make_ssa_name (word_type); - new_stmt = gimple_build_assign (signs, - BIT_XOR_EXPR, wvop0, wvop1); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - tree b_low = make_ssa_name (word_type); - new_stmt = gimple_build_assign (b_low, - BIT_AND_EXPR, wvop1, low_bits); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - tree a_low = make_ssa_name (word_type); - if (code == PLUS_EXPR) - new_stmt = gimple_build_assign (a_low, - BIT_AND_EXPR, wvop0, low_bits); - else - new_stmt = gimple_build_assign (a_low, - BIT_IOR_EXPR, wvop0, high_bits); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - if (code == MINUS_EXPR) + } + + tree result_low; + if (code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR) + { + unsigned int width = vector_element_bits (vectype); + tree inner_type = TREE_TYPE (vectype); + HOST_WIDE_INT max = GET_MODE_MASK (TYPE_MODE (inner_type)); + tree low_bits + = build_replicated_int_cst (word_type, width, max >> 1); + tree high_bits + = build_replicated_int_cst (word_type, + width, max & ~(max >> 1)); + tree signs; + if (code == PLUS_EXPR || code == MINUS_EXPR) + { + signs = make_ssa_name (word_type); + new_stmt = gimple_build_assign (signs, + BIT_XOR_EXPR, wvop0, wvop1); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + tree b_low = make_ssa_name (word_type); + new_stmt = gimple_build_assign (b_low, BIT_AND_EXPR, + wvop1, low_bits); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + tree a_low = make_ssa_name (word_type); + if (code == PLUS_EXPR) + new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR, + wvop0, low_bits); + else + new_stmt = gimple_build_assign (a_low, BIT_IOR_EXPR, + wvop0, high_bits); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + if (code == MINUS_EXPR) + { + new_stmt = gimple_build_assign (NULL_TREE, + BIT_NOT_EXPR, signs); + signs = make_ssa_name (word_type); + gimple_assign_set_lhs (new_stmt, signs); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); + } + new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR, + signs, high_bits); + signs = make_ssa_name (word_type); + gimple_assign_set_lhs (new_stmt, signs); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + result_low = make_ssa_name (word_type); + new_stmt = gimple_build_assign (result_low, code, + a_low, b_low); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + } + else /* if (code == NEGATE_EXPR) */ { - new_stmt = gimple_build_assign (NULL_TREE, - BIT_NOT_EXPR, signs); + tree a_low = make_ssa_name (word_type); + new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR, + wvop0, low_bits); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + signs = make_ssa_name (word_type); + new_stmt = gimple_build_assign (signs, BIT_NOT_EXPR, wvop0); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR, + signs, high_bits); signs = make_ssa_name (word_type); gimple_assign_set_lhs (new_stmt, signs); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + result_low = make_ssa_name (word_type); + new_stmt = gimple_build_assign (result_low, + MINUS_EXPR, high_bits, a_low); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } - new_stmt = gimple_build_assign (NULL_TREE, - BIT_AND_EXPR, signs, high_bits); - signs = make_ssa_name (word_type); - gimple_assign_set_lhs (new_stmt, signs); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (NULL_TREE, BIT_XOR_EXPR, + result_low, signs); result_low = make_ssa_name (word_type); - new_stmt = gimple_build_assign (result_low, code, a_low, b_low); + gimple_assign_set_lhs (new_stmt, result_low); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } else { - tree a_low = make_ssa_name (word_type); - new_stmt = gimple_build_assign (a_low, - BIT_AND_EXPR, wvop0, low_bits); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - signs = make_ssa_name (word_type); - new_stmt = gimple_build_assign (signs, BIT_NOT_EXPR, wvop0); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - new_stmt = gimple_build_assign (NULL_TREE, - BIT_AND_EXPR, signs, high_bits); - signs = make_ssa_name (word_type); - gimple_assign_set_lhs (new_stmt, signs); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (NULL_TREE, code, wvop0, wvop1); result_low = make_ssa_name (word_type); - new_stmt = gimple_build_assign (result_low, - MINUS_EXPR, high_bits, a_low); + gimple_assign_set_lhs (new_stmt, result_low); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + } - new_stmt = gimple_build_assign (NULL_TREE, BIT_XOR_EXPR, result_low, - signs); - result_low = make_ssa_name (word_type); - gimple_assign_set_lhs (new_stmt, result_low); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR, build1 (VIEW_CONVERT_EXPR, vectype, result_low)); @@ -7366,7 +7014,7 @@ vectorizable_operation (vec_info *vinfo, tree mask; if (masked_loop_p) mask = vect_get_loop_mask (loop_vinfo, gsi, masks, - vec_num * ncopies, vectype, i); + vec_num, vectype, i); else /* Dummy mask. */ mask = build_minus_one_cst (truth_type_for (vectype)); @@ -7393,7 +7041,7 @@ vectorizable_operation (vec_info *vinfo, if (len_loop_p) { tree len = vect_get_loop_len (loop_vinfo, gsi, lens, - vec_num * ncopies, vectype, i, 1); + vec_num, vectype, i, 1); signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); tree bias = build_int_cst (intQI_type_node, biasval); @@ -7420,21 +7068,19 @@ vectorizable_operation (vec_info *vinfo, && code == BIT_AND_EXPR && VECTOR_BOOLEAN_TYPE_P (vectype)) { - if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, - ncopies})) + if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, vec_num })) { mask = vect_get_loop_mask (loop_vinfo, gsi, masks, - vec_num * ncopies, vectype, i); + vec_num, vectype, i); vop0 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask, vop0, gsi); } - if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, - ncopies })) + if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, vec_num })) { mask = vect_get_loop_mask (loop_vinfo, gsi, masks, - vec_num * ncopies, vectype, i); + vec_num, vectype, i); vop1 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask, vop1, gsi); @@ -7465,15 +7111,9 @@ vectorizable_operation (vec_info *vinfo, new_stmt, gsi); } - if (slp_node) - slp_node->push_vec_def (new_stmt); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + slp_node->push_vec_def (new_stmt); } - if (!slp_node) - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - vec_oprnds0.release (); vec_oprnds1.release (); vec_oprnds2.release (); @@ -7730,7 +7370,8 @@ scan_store_can_perm_p (tree vectype, tree init, static bool check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype, - enum vect_def_type rhs_dt, slp_tree slp_node, tree mask, + enum vect_def_type rhs_dt, slp_tree slp_node, + slp_tree mask_node, vect_memory_access_type memory_access_type) { loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); @@ -7738,13 +7379,14 @@ check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype, tree ref_type; gcc_assert (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1); - if ((slp_node && SLP_TREE_LANES (slp_node) > 1) - || mask + if (SLP_TREE_LANES (slp_node) > 1 + || mask_node || memory_access_type != VMAT_CONTIGUOUS || TREE_CODE (DR_BASE_ADDRESS (dr_info->dr)) != ADDR_EXPR || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0)) || loop_vinfo == NULL || LOOP_VINFO_FULLY_MASKED_P (loop_vinfo) + || LOOP_VINFO_EPILOGUE_P (loop_vinfo) || STMT_VINFO_GROUPED_ACCESS (stmt_info) || !integer_zerop (get_dr_vinfo_offset (vinfo, dr_info)) || !integer_zerop (DR_INIT (dr_info->dr)) @@ -8134,17 +7776,16 @@ check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype, static bool vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, - slp_tree slp_node, gimple_stmt_iterator *gsi, - gimple **vec_stmt, int ncopies) + slp_tree slp_node, gimple_stmt_iterator *gsi) { loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info); tree ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)); - tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vectype = SLP_TREE_VECTYPE (slp_node); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, - "transform scan store. ncopies = %d\n", ncopies); + "transform scan store.\n"); gimple *stmt = STMT_VINFO_STMT (stmt_info); tree rhs = gimple_assign_rhs1 (stmt); @@ -8235,13 +7876,19 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, perms[i] = vect_gen_perm_mask_checked (vectype, indices); } + vec_loop_lens *loop_lens + = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo) + ? &LOOP_VINFO_LENS (loop_vinfo) + : NULL); + tree vec_oprnd1 = NULL_TREE; tree vec_oprnd2 = NULL_TREE; tree vec_oprnd3 = NULL_TREE; tree dataref_ptr = DR_BASE_ADDRESS (dr_info->dr); tree dataref_offset = build_int_cst (ref_type, 0); tree bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, - vectype, VMAT_CONTIGUOUS); + vectype, VMAT_CONTIGUOUS, + loop_lens); tree ldataref_ptr = NULL_TREE; tree orig = NULL_TREE; if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store) @@ -8255,8 +7902,8 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, /* We want to lookup the vector operands of the reduction, not those of the store - for SLP we have to use the proper SLP node for the lookup, which should be the single child of the scan store. */ - vect_get_vec_defs (vinfo, stmt_info, SLP_TREE_CHILDREN (slp_node)[0], - ncopies, rhs1, &vec_oprnds2, rhs2, &vec_oprnds3); + vect_get_vec_defs (vinfo, SLP_TREE_CHILDREN (slp_node)[0], + rhs1, &vec_oprnds2, rhs2, &vec_oprnds3); /* ??? For SLP we do not key the def on 'rhs1' or 'rhs2' but get them in SLP child order. So we have to swap here with logic similar to above. */ @@ -8270,7 +7917,7 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, std::swap (vec_oprnds2[i], vec_oprnds3[i]);; } else - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, + vect_get_vec_defs (vinfo, slp_node, rhs2, &vec_oprnds3); for (unsigned j = 0; j < vec_oprnds3.length (); j++) { @@ -8291,11 +7938,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr)); gimple *g = gimple_build_assign (vec_oprnd2, data_ref); vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - if (! slp_node) - { - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - } } tree v = vec_oprnd2; @@ -8309,11 +7951,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, ? zero_vec : vec_oprnd1, v, perms[i]); vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - if (! slp_node) - { - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - } if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond) { @@ -8330,8 +7967,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, new_temp, vec_oprnd1); vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - if (! slp_node) - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); new_temp = new_temp2; } @@ -8349,8 +7984,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree new_temp2 = make_ssa_name (vectype); g = gimple_build_assign (new_temp2, code, v, new_temp); vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - if (! slp_node) - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); v = new_temp2; } @@ -8358,8 +7991,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree new_temp = make_ssa_name (vectype); gimple *g = gimple_build_assign (new_temp, code, orig, v); vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - if (! slp_node) - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); tree last_perm_arg = new_temp; /* For exclusive scan, new_temp computed above is the exclusive scan @@ -8370,16 +8001,12 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, last_perm_arg = make_ssa_name (vectype); g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2); vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - if (! slp_node) - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); } orig = make_ssa_name (vectype); g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg, last_perm_arg, perms[units_log2]); vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - if (! slp_node) - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); if (!inscan_var_store) { @@ -8389,8 +8016,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, vect_copy_ref_info (data_ref, DR_REF (dr_info->dr)); g = gimple_build_assign (data_ref, new_temp); vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - if (! slp_node) - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); } } @@ -8406,8 +8031,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, vect_copy_ref_info (data_ref, DR_REF (dr_info->dr)); gimple *g = gimple_build_assign (data_ref, orig); vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - if (! slp_node) - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); } return true; } @@ -8417,14 +8040,15 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure) that can be vectorized. - If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized - stmt to replace it, put it in VEC_STMT, and insert it at GSI. + If COST_VEC is passed, calculate costs but don't change anything, + otherwise, vectorize STMT_INFO: create a vectorized stmt to replace + it, and insert it at GSI. Return true if STMT_INFO is vectorizable in this way. */ static bool vectorizable_store (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - gimple **vec_stmt, slp_tree slp_node, + slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree data_ref; @@ -8439,16 +8063,13 @@ vectorizable_store (vec_info *vinfo, tree dataref_ptr = NULL_TREE; tree dataref_offset = NULL_TREE; gimple *ptr_incr = NULL; - int ncopies; int j; stmt_vec_info first_stmt_info; bool grouped_store; unsigned int group_size, i; - bool slp = (slp_node != NULL); unsigned int vec_num; bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo); tree aggr_type; - gather_scatter_info gs_info; poly_uint64 vf; vec_load_store_type vls_type; tree ref_type; @@ -8457,12 +8078,12 @@ vectorizable_store (vec_info *vinfo, return false; if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def - && ! vec_stmt) + && cost_vec) return false; /* Is vectorizable store? */ - tree mask = NULL_TREE, mask_vectype = NULL_TREE; + tree mask_vectype = NULL_TREE; slp_tree mask_node = NULL; if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt)) { @@ -8490,21 +8111,17 @@ vectorizable_store (vec_info *vinfo, return false; int mask_index = internal_fn_mask_index (ifn); - if (mask_index >= 0 && slp_node) + if (mask_index >= 0) mask_index = vect_slp_child_index_for_operand (call, mask_index, STMT_VINFO_GATHER_SCATTER_P (stmt_info)); if (mask_index >= 0 - && !vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_index, - &mask, &mask_node, &mask_dt, + && !vect_check_scalar_mask (vinfo, slp_node, mask_index, + &mask_node, &mask_dt, &mask_vectype)) return false; } - /* Cannot have hybrid store SLP -- that would mean storing to the - same location twice. */ - gcc_assert (slp == PURE_SLP_STMT (stmt_info)); - - tree vectype = STMT_VINFO_VECTYPE (stmt_info), rhs_vectype = NULL_TREE; + tree vectype = SLP_TREE_VECTYPE (slp_node), rhs_vectype = NULL_TREE; poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); if (loop_vinfo) @@ -8514,21 +8131,12 @@ vectorizable_store (vec_info *vinfo, } else vf = 1; - - /* Multiple types in SLP are handled by creating the appropriate number of - vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in - case of SLP. */ - if (slp) - ncopies = 1; - else - ncopies = vect_get_num_copies (loop_vinfo, vectype); - - gcc_assert (ncopies >= 1); + vec_num = vect_get_num_copies (vinfo, slp_node); /* FORNOW. This restriction should be relaxed. */ if (loop && nested_in_vect_loop_p (loop, stmt_info) - && (ncopies > 1 || (slp && SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) > 1))) + && vec_num > 1) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -8536,10 +8144,9 @@ vectorizable_store (vec_info *vinfo, return false; } - tree op; slp_tree op_node; if (!vect_check_store_rhs (vinfo, stmt_info, slp_node, - &op, &op_node, &rhs_dt, &rhs_vectype, &vls_type)) + &op_node, &rhs_dt, &rhs_vectype, &vls_type)) return false; elem_type = TREE_TYPE (vectype); @@ -8548,19 +8155,21 @@ vectorizable_store (vec_info *vinfo, if (!STMT_VINFO_DATA_REF (stmt_info)) return false; - vect_memory_access_type memory_access_type; - enum dr_alignment_support alignment_support_scheme; - int misalignment; - poly_int64 poffset; - internal_fn lanes_ifn; - if (!get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask, vls_type, - ncopies, &memory_access_type, &poffset, - &alignment_support_scheme, &misalignment, &gs_info, - &lanes_ifn)) + vect_load_store_data _ls_data{}; + vect_load_store_data &ls = slp_node->get_data (_ls_data); + if (cost_vec + && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node, + vls_type, &_ls_data)) return false; - - if (slp_node - && slp_node->ldst_lanes + /* Temporary aliases to analysis data, should not be modified through + these. */ + const vect_memory_access_type memory_access_type = ls.memory_access_type; + const dr_alignment_support alignment_support_scheme + = ls.alignment_support_scheme; + const int misalignment = ls.misalignment; + const poly_int64 poffset = ls.poffset; + + if (slp_node->ldst_lanes && memory_access_type != VMAT_LOAD_STORE_LANES) { if (dump_enabled_p ()) @@ -8569,7 +8178,7 @@ vectorizable_store (vec_info *vinfo, return false; } - if (mask) + if (mask_node) { if (memory_access_type == VMAT_CONTIGUOUS) { @@ -8579,17 +8188,16 @@ vectorizable_store (vec_info *vinfo, return false; } else if (memory_access_type != VMAT_LOAD_STORE_LANES - && (memory_access_type != VMAT_GATHER_SCATTER - || (gs_info.decl && !VECTOR_BOOLEAN_TYPE_P (mask_vectype)))) + && (!mat_gather_scatter_p (memory_access_type) + || (memory_access_type == VMAT_GATHER_SCATTER_LEGACY + && !VECTOR_BOOLEAN_TYPE_P (mask_vectype)))) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, "unsupported access type for masked store.\n"); return false; } - else if (memory_access_type == VMAT_GATHER_SCATTER - && gs_info.ifn == IFN_LAST - && !gs_info.decl) + else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -8607,8 +8215,7 @@ vectorizable_store (vec_info *vinfo, dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL; grouped_store = (STMT_VINFO_GROUPED_ACCESS (stmt_info) - && memory_access_type != VMAT_GATHER_SCATTER - && (slp || memory_access_type != VMAT_CONTIGUOUS)); + && !mat_gather_scatter_p (memory_access_type)); if (grouped_store) { first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info); @@ -8619,35 +8226,29 @@ vectorizable_store (vec_info *vinfo, { first_stmt_info = stmt_info; first_dr_info = dr_info; - group_size = vec_num = 1; + group_size = 1; } - if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && !vec_stmt) + if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && cost_vec) { - if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp_node, mask, - memory_access_type)) + if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp_node, + mask_node, memory_access_type)) return false; } - bool costing_p = !vec_stmt; + bool costing_p = cost_vec; if (costing_p) /* transformation not required. */ { - STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type; - if (slp_node) - SLP_TREE_MEMORY_ACCESS_TYPE (slp_node) = memory_access_type; - if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)) check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node, - vls_type, group_size, - memory_access_type, &gs_info, - mask); + vls_type, group_size, &ls, + mask_node); - if (slp_node - && (!vect_maybe_update_slp_op_vectype (op_node, vectype) - || (mask - && !vect_maybe_update_slp_op_vectype (mask_node, - mask_vectype)))) + if (!vect_maybe_update_slp_op_vectype (op_node, vectype) + || (mask_node + && !vect_maybe_update_slp_op_vectype (mask_node, + mask_vectype))) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -8657,30 +8258,15 @@ vectorizable_store (vec_info *vinfo, if (dump_enabled_p () && memory_access_type != VMAT_ELEMENTWISE - && memory_access_type != VMAT_GATHER_SCATTER && memory_access_type != VMAT_STRIDED_SLP && memory_access_type != VMAT_INVARIANT && alignment_support_scheme != dr_aligned) dump_printf_loc (MSG_NOTE, vect_location, "Vectorizing an unaligned access.\n"); - STMT_VINFO_TYPE (stmt_info) = store_vec_info_type; - - /* As function vect_transform_stmt shows, for interleaving stores - the whole chain is vectorized when the last store in the chain - is reached, the other stores in the group are skipped. So we - want to only cost the last one here, but it's not trivial to - get the last, as it's equivalent to use the first one for - costing, use the first one instead. */ - if (grouped_store - && !slp - && first_stmt_info != stmt_info) - return true; + SLP_TREE_TYPE (slp_node) = store_vec_info_type; + slp_node->data = new vect_load_store_data (std::move (ls)); } - if (slp_node) - gcc_assert (memory_access_type == SLP_TREE_MEMORY_ACCESS_TYPE (stmt_info)); - else - gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info)); /* Transform. */ @@ -8689,14 +8275,14 @@ vectorizable_store (vec_info *vinfo, if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3) { gcc_assert (memory_access_type == VMAT_CONTIGUOUS); - gcc_assert (!slp || SLP_TREE_LANES (slp_node) == 1); + gcc_assert (SLP_TREE_LANES (slp_node) == 1); if (costing_p) { unsigned int inside_cost = 0, prologue_cost = 0; if (vls_type == VLS_STORE_INVARIANT) prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec, - stmt_info, 0, vect_prologue); - vect_get_store_cost (vinfo, stmt_info, slp_node, ncopies, + slp_node, 0, vect_prologue); + vect_get_store_cost (vinfo, stmt_info, slp_node, 1, alignment_support_scheme, misalignment, &inside_cost, cost_vec); @@ -8708,68 +8294,24 @@ vectorizable_store (vec_info *vinfo, return true; } - return vectorizable_scan_store (vinfo, stmt_info, slp_node, - gsi, vec_stmt, ncopies); + return vectorizable_scan_store (vinfo, stmt_info, slp_node, gsi); } - if (grouped_store || slp) - { - /* FORNOW */ - gcc_assert (!grouped_store - || !loop - || !nested_in_vect_loop_p (loop, stmt_info)); + /* FORNOW */ + gcc_assert (!grouped_store + || !loop + || !nested_in_vect_loop_p (loop, stmt_info)); - if (slp) - { - grouped_store = false; - /* VEC_NUM is the number of vect stmts to be created for this - group. */ - vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); - first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0]; - gcc_assert (!STMT_VINFO_GROUPED_ACCESS (first_stmt_info) - || (DR_GROUP_FIRST_ELEMENT (first_stmt_info) - == first_stmt_info)); - first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info); - op = vect_get_store_rhs (first_stmt_info); - } - else - /* VEC_NUM is the number of vect stmts to be created for this - group. */ - vec_num = group_size; + grouped_store = false; + first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0]; + gcc_assert (!STMT_VINFO_GROUPED_ACCESS (first_stmt_info) + || (DR_GROUP_FIRST_ELEMENT (first_stmt_info) == first_stmt_info)); + first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info); - ref_type = get_group_alias_ptr_type (first_stmt_info); - } - else - ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr)); + ref_type = get_group_alias_ptr_type (first_stmt_info); if (!costing_p && dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, "transform store. ncopies = %d\n", - ncopies); - - /* Check if we need to update prologue cost for invariant, - and update it accordingly if so. If it's not for - interleaving store, we can just check vls_type; but if - it's for interleaving store, need to check the def_type - of the stored value since the current vls_type is just - for first_stmt_info. */ - auto update_prologue_cost = [&](unsigned *prologue_cost, tree store_rhs) - { - gcc_assert (costing_p); - if (slp) - return; - if (grouped_store) - { - gcc_assert (store_rhs); - enum vect_def_type cdt; - gcc_assert (vect_is_simple_use (store_rhs, vinfo, &cdt)); - if (cdt != vect_constant_def && cdt != vect_external_def) - return; - } - else if (vls_type != VLS_STORE_INVARIANT) - return; - *prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec, stmt_info, - slp_node, 0, vect_prologue); - }; + dump_printf_loc (MSG_NOTE, vect_location, "transform store.\n"); if (memory_access_type == VMAT_ELEMENTWISE || memory_access_type == VMAT_STRIDED_SLP) @@ -8777,14 +8319,12 @@ vectorizable_store (vec_info *vinfo, unsigned inside_cost = 0, prologue_cost = 0; gimple_stmt_iterator incr_gsi; bool insert_after; - gimple *incr; tree offvar = NULL_TREE; tree ivstep; tree running_off; tree stride_base, stride_step, alias_off; tree vec_oprnd = NULL_TREE; tree dr_offset; - unsigned int g; /* Checked by get_load_store_type. */ unsigned int const_nunits = nunits.to_constant (); @@ -8818,114 +8358,124 @@ vectorizable_store (vec_info *vinfo, ... */ + /* ??? Modify local copies of alignment_support_scheme and + misalignment, but this part of analysis should be done + earlier and remembered, likewise the chosen load mode. */ + const dr_alignment_support tem = alignment_support_scheme; + dr_alignment_support alignment_support_scheme = tem; + const int tem2 = misalignment; + int misalignment = tem2; + unsigned nstores = const_nunits; unsigned lnel = 1; tree ltype = elem_type; tree lvectype = vectype; - if (slp) - { - HOST_WIDE_INT n = gcd (group_size, const_nunits); - if (n == const_nunits) - { - int mis_align = dr_misalignment (first_dr_info, vectype); - dr_alignment_support dr_align - = vect_supportable_dr_alignment (vinfo, dr_info, vectype, - mis_align); - if (dr_align == dr_aligned - || dr_align == dr_unaligned_supported) - { - nstores = 1; - lnel = const_nunits; - ltype = vectype; - lvectype = vectype; - alignment_support_scheme = dr_align; - misalignment = mis_align; - } - } - else if (n > 1) - { - nstores = const_nunits / n; - lnel = n; - ltype = build_vector_type (elem_type, n); + HOST_WIDE_INT n = gcd (group_size, const_nunits); + if (n == const_nunits) + { + int mis_align = dr_misalignment (first_dr_info, vectype); + /* With VF > 1 we advance the DR by step, if that is constant + and only aligned when performed VF times, DR alignment + analysis can analyze this as aligned since it assumes + contiguous accesses. But that is not how we code generate + here, so adjust for this. */ + if (maybe_gt (vf, 1u) + && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr), + DR_TARGET_ALIGNMENT (first_dr_info))) + mis_align = -1; + dr_alignment_support dr_align + = vect_supportable_dr_alignment (vinfo, dr_info, vectype, + mis_align); + if (dr_align == dr_aligned + || dr_align == dr_unaligned_supported) + { + nstores = 1; + lnel = const_nunits; + ltype = vectype; lvectype = vectype; - int mis_align = dr_misalignment (first_dr_info, ltype); - dr_alignment_support dr_align - = vect_supportable_dr_alignment (vinfo, dr_info, ltype, - mis_align); alignment_support_scheme = dr_align; misalignment = mis_align; - - /* First check if vec_extract optab doesn't support extraction - of vector elts directly. */ - scalar_mode elmode = SCALAR_TYPE_MODE (elem_type); - machine_mode vmode; - if (!VECTOR_MODE_P (TYPE_MODE (vectype)) - || !related_vector_mode (TYPE_MODE (vectype), elmode, - n).exists (&vmode) - || (convert_optab_handler (vec_extract_optab, - TYPE_MODE (vectype), vmode) - == CODE_FOR_nothing) - || !(dr_align == dr_aligned - || dr_align == dr_unaligned_supported)) - { - /* Try to avoid emitting an extract of vector elements - by performing the extracts using an integer type of the - same size, extracting from a vector of those and then - re-interpreting it as the original vector type if - supported. */ - unsigned lsize - = n * GET_MODE_BITSIZE (elmode); - unsigned int lnunits = const_nunits / n; - /* If we can't construct such a vector fall back to - element extracts from the original vector type and - element size stores. */ - if (int_mode_for_size (lsize, 0).exists (&elmode) - && VECTOR_MODE_P (TYPE_MODE (vectype)) - && related_vector_mode (TYPE_MODE (vectype), elmode, - lnunits).exists (&vmode) - && (convert_optab_handler (vec_extract_optab, - vmode, elmode) - != CODE_FOR_nothing)) - { - nstores = lnunits; - lnel = n; - ltype = build_nonstandard_integer_type (lsize, 1); - lvectype = build_vector_type (ltype, nstores); - } - /* Else fall back to vector extraction anyway. - Fewer stores are more important than avoiding spilling - of the vector we extract from. Compared to the - construction case in vectorizable_load no store-forwarding - issue exists here for reasonable archs. But only - if the store is supported. */ - else if (!(dr_align == dr_aligned - || dr_align == dr_unaligned_supported)) - { - nstores = const_nunits; - lnel = 1; - ltype = elem_type; - lvectype = vectype; - } - } } - unsigned align; - /* ??? We'd want to use - if (alignment_support_scheme == dr_aligned) - align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info)); - since doing that is what we assume we can in the above checks. - But this interferes with groups with gaps where for example - VF == 2 makes the group in the unrolled loop aligned but the - fact that we advance with step between the two subgroups - makes the access to the second unaligned. See PR119586. - We have to anticipate that here or adjust code generation to - avoid the misaligned loads by means of permutations. */ - align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info)); - /* Alignment is at most the access size if we do multiple stores. */ - if (nstores > 1) - align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align); - ltype = build_aligned_type (ltype, align * BITS_PER_UNIT); - ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); } + else if (n > 1) + { + nstores = const_nunits / n; + lnel = n; + ltype = build_vector_type (elem_type, n); + lvectype = vectype; + int mis_align = dr_misalignment (first_dr_info, ltype); + if (maybe_gt (vf, 1u) + && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr), + DR_TARGET_ALIGNMENT (first_dr_info))) + mis_align = -1; + dr_alignment_support dr_align + = vect_supportable_dr_alignment (vinfo, dr_info, ltype, + mis_align); + alignment_support_scheme = dr_align; + misalignment = mis_align; + + /* First check if vec_extract optab doesn't support extraction + of vector elts directly. */ + scalar_mode elmode = SCALAR_TYPE_MODE (elem_type); + machine_mode vmode; + if (!VECTOR_MODE_P (TYPE_MODE (vectype)) + || !related_vector_mode (TYPE_MODE (vectype), elmode, + n).exists (&vmode) + || (convert_optab_handler (vec_extract_optab, + TYPE_MODE (vectype), vmode) + == CODE_FOR_nothing) + || !(dr_align == dr_aligned + || dr_align == dr_unaligned_supported)) + { + /* Try to avoid emitting an extract of vector elements + by performing the extracts using an integer type of the + same size, extracting from a vector of those and then + re-interpreting it as the original vector type if + supported. */ + unsigned lsize = n * GET_MODE_BITSIZE (elmode); + unsigned int lnunits = const_nunits / n; + /* If we can't construct such a vector fall back to + element extracts from the original vector type and + element size stores. */ + if (int_mode_for_size (lsize, 0).exists (&elmode) + && VECTOR_MODE_P (TYPE_MODE (vectype)) + && related_vector_mode (TYPE_MODE (vectype), elmode, + lnunits).exists (&vmode) + && (convert_optab_handler (vec_extract_optab, + vmode, elmode) + != CODE_FOR_nothing)) + { + nstores = lnunits; + lnel = n; + ltype = build_nonstandard_integer_type (lsize, 1); + lvectype = build_vector_type (ltype, nstores); + } + /* Else fall back to vector extraction anyway. + Fewer stores are more important than avoiding spilling + of the vector we extract from. Compared to the + construction case in vectorizable_load no store-forwarding + issue exists here for reasonable archs. But only + if the store is supported. */ + else if (!(dr_align == dr_aligned + || dr_align == dr_unaligned_supported)) + { + nstores = const_nunits; + lnel = 1; + ltype = elem_type; + lvectype = vectype; + } + } + } + unsigned align; + if (alignment_support_scheme == dr_aligned) + align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info)); + else + align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info)); + /* Alignment is at most the access size if we do multiple stores. */ + if (nstores > 1) + align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align); + ltype = build_aligned_type (ltype, align * BITS_PER_UNIT); + int ncopies = vec_num; if (!costing_p) { @@ -8939,115 +8489,76 @@ vectorizable_store (vec_info *vinfo, ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep); create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi, insert_after, &offvar, NULL); - incr = gsi_stmt (incr_gsi); stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step); } alias_off = build_int_cst (ref_type, 0); - stmt_vec_info next_stmt_info = first_stmt_info; auto_vec<tree> vec_oprnds; /* For costing some adjacent vector stores, we'd like to cost with the total number of them once instead of cost each one by one. */ unsigned int n_adjacent_stores = 0; - for (g = 0; g < group_size; g++) + running_off = offvar; + if (!costing_p) + vect_get_slp_defs (op_node, &vec_oprnds); + unsigned int group_el = 0; + unsigned HOST_WIDE_INT elsz + = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype))); + for (j = 0; j < ncopies; j++) { - running_off = offvar; if (!costing_p) { - if (g) + vec_oprnd = vec_oprnds[j]; + /* Pun the vector to extract from if necessary. */ + if (lvectype != vectype) { - tree size = TYPE_SIZE_UNIT (ltype); - tree pos - = fold_build2 (MULT_EXPR, sizetype, size_int (g), size); - tree newoff = copy_ssa_name (running_off, NULL); - incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR, - running_off, pos); - vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi); - running_off = newoff; + tree tem = make_ssa_name (lvectype); + tree cvt = build1 (VIEW_CONVERT_EXPR, lvectype, vec_oprnd); + gimple *pun = gimple_build_assign (tem, cvt); + vect_finish_stmt_generation (vinfo, stmt_info, pun, gsi); + vec_oprnd = tem; } } - if (!slp) - op = vect_get_store_rhs (next_stmt_info); - if (!costing_p) - vect_get_vec_defs (vinfo, next_stmt_info, slp_node, ncopies, op, - &vec_oprnds); - else - update_prologue_cost (&prologue_cost, op); - unsigned int group_el = 0; - unsigned HOST_WIDE_INT - elsz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype))); - for (j = 0; j < ncopies; j++) + for (i = 0; i < nstores; i++) { - if (!costing_p) + if (costing_p) { - vec_oprnd = vec_oprnds[j]; - /* Pun the vector to extract from if necessary. */ - if (lvectype != vectype) - { - tree tem = make_ssa_name (lvectype); - tree cvt - = build1 (VIEW_CONVERT_EXPR, lvectype, vec_oprnd); - gimple *pun = gimple_build_assign (tem, cvt); - vect_finish_stmt_generation (vinfo, stmt_info, pun, gsi); - vec_oprnd = tem; - } + n_adjacent_stores++; + continue; } - for (i = 0; i < nstores; i++) + tree newref, newoff; + gimple *incr, *assign; + tree size = TYPE_SIZE (ltype); + /* Extract the i'th component. */ + tree pos = fold_build2 (MULT_EXPR, bitsizetype, + bitsize_int (i), size); + tree elem = fold_build3 (BIT_FIELD_REF, ltype, vec_oprnd, + size, pos); + + elem = force_gimple_operand_gsi (gsi, elem, true, NULL_TREE, true, + GSI_SAME_STMT); + + tree this_off = build_int_cst (TREE_TYPE (alias_off), + group_el * elsz); + newref = build2 (MEM_REF, ltype, running_off, this_off); + vect_copy_ref_info (newref, DR_REF (first_dr_info->dr)); + + /* And store it to *running_off. */ + assign = gimple_build_assign (newref, elem); + vect_finish_stmt_generation (vinfo, stmt_info, assign, gsi); + + group_el += lnel; + if (group_el == group_size) { - if (costing_p) - { - n_adjacent_stores++; - continue; - } - tree newref, newoff; - gimple *incr, *assign; - tree size = TYPE_SIZE (ltype); - /* Extract the i'th component. */ - tree pos = fold_build2 (MULT_EXPR, bitsizetype, - bitsize_int (i), size); - tree elem = fold_build3 (BIT_FIELD_REF, ltype, vec_oprnd, - size, pos); - - elem = force_gimple_operand_gsi (gsi, elem, true, - NULL_TREE, true, - GSI_SAME_STMT); - - tree this_off = build_int_cst (TREE_TYPE (alias_off), - group_el * elsz); - newref = build2 (MEM_REF, ltype, - running_off, this_off); - vect_copy_ref_info (newref, DR_REF (first_dr_info->dr)); - - /* And store it to *running_off. */ - assign = gimple_build_assign (newref, elem); - vect_finish_stmt_generation (vinfo, stmt_info, assign, gsi); - - group_el += lnel; - if (! slp - || group_el == group_size) - { - newoff = copy_ssa_name (running_off, NULL); - incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR, - running_off, stride_step); - vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi); + newoff = copy_ssa_name (running_off, NULL); + incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR, + running_off, stride_step); + vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi); - running_off = newoff; - group_el = 0; - } - if (g == group_size - 1 - && !slp) - { - if (j == 0 && i == 0) - *vec_stmt = assign; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (assign); - } + running_off = newoff; + group_el = 0; } } - next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info); - vec_oprnds.truncate(0); - if (slp) - break; } if (costing_p) @@ -9064,14 +8575,13 @@ vectorizable_store (vec_info *vinfo, else inside_cost += record_stmt_cost (cost_vec, n_adjacent_stores, - scalar_store, stmt_info, 0, vect_body); + scalar_store, slp_node, 0, vect_body); /* Only need vector extracting when there are more than one stores. */ if (nstores > 1) inside_cost += record_stmt_cost (cost_vec, n_adjacent_stores, - vec_to_scalar, stmt_info, slp_node, - 0, vect_body); + vec_to_scalar, slp_node, 0, vect_body); } if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -9105,7 +8615,7 @@ vectorizable_store (vec_info *vinfo, realignment. vect_supportable_dr_alignment always returns either dr_aligned or dr_unaligned_supported for masked operations. */ gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES - && !mask + && !mask_node && !loop_masks) || alignment_support_scheme == dr_aligned || alignment_support_scheme == dr_unaligned_supported); @@ -9121,12 +8631,17 @@ vectorizable_store (vec_info *vinfo, aggr_type = NULL_TREE; bump = NULL_TREE; } - else if (memory_access_type == VMAT_GATHER_SCATTER) + else if (mat_gather_scatter_p (memory_access_type)) { aggr_type = elem_type; if (!costing_p) - vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info, - &bump, &vec_offset, loop_lens); + { + tree vtype = ls.ls_type ? ls.ls_type : vectype; + vect_get_strided_load_store_ops (stmt_info, slp_node, vtype, + ls.strided_offset_vectype, + loop_vinfo, gsi, + &bump, &vec_offset, loop_lens); + } } else { @@ -9139,7 +8654,7 @@ vectorizable_store (vec_info *vinfo, memory_access_type, loop_lens); } - if (mask && !costing_p) + if (mask_node && !costing_p) LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true; /* In case the vectorization factor (VF) is bigger than the number @@ -9147,39 +8662,6 @@ vectorizable_store (vec_info *vinfo, more than one vector stmt - i.e - we need to "unroll" the vector stmt by a factor VF/nunits. */ - /* In case of interleaving (non-unit grouped access): - - S1: &base + 2 = x2 - S2: &base = x0 - S3: &base + 1 = x1 - S4: &base + 3 = x3 - - We create vectorized stores starting from base address (the access of the - first stmt in the chain (S2 in the above example), when the last store stmt - of the chain (S4) is reached: - - VS1: &base = vx2 - VS2: &base + vec_size*1 = vx0 - VS3: &base + vec_size*2 = vx1 - VS4: &base + vec_size*3 = vx3 - - Then permutation statements are generated: - - VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} > - VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} > - ... - - And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts - (the order of the data-refs in the output of vect_permute_store_chain - corresponds to the order of scalar stmts in the interleaving chain - see - the documentation of vect_permute_store_chain()). - - In case of both multiple types and interleaving, above vector stores and - permutation stmts are created for every copy. The result vector stmts are - put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding - STMT_VINFO_RELATED_STMT for the next copies. - */ - auto_vec<tree> dr_chain (group_size); auto_vec<tree> vec_masks; tree vec_mask = NULL; @@ -9189,7 +8671,9 @@ vectorizable_store (vec_info *vinfo, if (memory_access_type == VMAT_LOAD_STORE_LANES) { - if (costing_p && slp_node) + const internal_fn lanes_ifn = ls.lanes_ifn; + + if (costing_p) /* Update all incoming store operand nodes, the general handling above only handles the mask and the first store operand node. */ for (slp_tree child : SLP_TREE_CHILDREN (slp_node)) @@ -9205,49 +8689,18 @@ vectorizable_store (vec_info *vinfo, /* For costing some adjacent vector stores, we'd like to cost with the total number of them once instead of cost each one by one. */ unsigned int n_adjacent_stores = 0; - if (slp) - ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) / group_size; + int ncopies = vec_num / group_size; for (j = 0; j < ncopies; j++) { - gimple *new_stmt; if (j == 0) { - /* For interleaved stores we collect vectorized defs for all - the stores in the group in DR_CHAIN. DR_CHAIN is then used - as an input to vect_permute_store_chain(). */ - stmt_vec_info next_stmt_info = first_stmt_info; - for (i = 0; i < group_size; i++) - { - /* Since gaps are not supported for interleaved stores, - DR_GROUP_SIZE is the exact number of stmts in the - chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE. */ - op = vect_get_store_rhs (next_stmt_info); - if (costing_p) - update_prologue_cost (&prologue_cost, op); - else if (!slp) - { - vect_get_vec_defs_for_operand (vinfo, next_stmt_info, - ncopies, op, - gvec_oprnds[i]); - vec_oprnd = (*gvec_oprnds[i])[0]; - dr_chain.quick_push (vec_oprnd); - } - next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info); - } - if (!costing_p) { - if (mask) + if (mask_node) { - if (slp_node) - vect_get_slp_defs (mask_node, &vec_masks); - else - vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies, - mask, &vec_masks, - mask_vectype); + vect_get_slp_defs (mask_node, &vec_masks); vec_mask = vec_masks[0]; } - dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type, NULL, offset, &dummy, @@ -9257,20 +8710,7 @@ vectorizable_store (vec_info *vinfo, else if (!costing_p) { gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)); - /* DR_CHAIN is then used as an input to - vect_permute_store_chain(). */ - if (!slp) - { - /* We should have caught mismatched types earlier. */ - gcc_assert ( - useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd))); - for (i = 0; i < group_size; i++) - { - vec_oprnd = (*gvec_oprnds[i])[j]; - dr_chain[i] = vec_oprnd; - } - } - if (mask) + if (mask_node) vec_mask = vec_masks[j]; dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, stmt_info, bump); @@ -9293,17 +8733,12 @@ vectorizable_store (vec_info *vinfo, /* Store the individual vectors into the array. */ for (i = 0; i < group_size; i++) { - if (slp) - { - slp_tree child; - if (i == 0 || !mask_node) - child = SLP_TREE_CHILDREN (slp_node)[i]; - else - child = SLP_TREE_CHILDREN (slp_node)[i + 1]; - vec_oprnd = SLP_TREE_VEC_DEFS (child)[j]; - } + slp_tree child; + if (i == 0 || !mask_node) + child = SLP_TREE_CHILDREN (slp_node)[i]; else - vec_oprnd = dr_chain[i]; + child = SLP_TREE_CHILDREN (slp_node)[i + 1]; + vec_oprnd = SLP_TREE_VEC_DEFS (child)[j]; write_vector_array (vinfo, stmt_info, gsi, vec_oprnd, vec_array, i); } @@ -9369,14 +8804,9 @@ vectorizable_store (vec_info *vinfo, } gimple_call_set_nothrow (call, true); vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); - new_stmt = call; /* Record that VEC_ARRAY is now dead. */ vect_clobber_variable (vinfo, stmt_info, gsi, vec_array); - if (j == 0 && !slp) - *vec_stmt = new_stmt; - if (!slp) - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (costing_p) @@ -9395,12 +8825,14 @@ vectorizable_store (vec_info *vinfo, return true; } - if (memory_access_type == VMAT_GATHER_SCATTER) + if (mat_gather_scatter_p (memory_access_type)) { - gcc_assert (!grouped_store); + gcc_assert (!grouped_store || ls.ls_type); + if (ls.ls_type) + vectype = ls.ls_type; auto_vec<tree> vec_offsets; unsigned int inside_cost = 0, prologue_cost = 0; - int num_stmts = ncopies * vec_num; + int num_stmts = vec_num; for (j = 0; j < num_stmts; j++) { gimple *new_stmt; @@ -9408,32 +8840,18 @@ vectorizable_store (vec_info *vinfo, { if (costing_p && vls_type == VLS_STORE_INVARIANT) prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec, - stmt_info, slp_node, 0, - vect_prologue); + slp_node, 0, vect_prologue); else if (!costing_p) { /* Since the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is of size 1. */ gcc_assert (group_size == 1); - if (slp_node) - vect_get_slp_defs (op_node, gvec_oprnds[0]); - else - vect_get_vec_defs_for_operand (vinfo, first_stmt_info, - num_stmts, op, gvec_oprnds[0]); - if (mask) - { - if (slp_node) - vect_get_slp_defs (mask_node, &vec_masks); - else - vect_get_vec_defs_for_operand (vinfo, stmt_info, - num_stmts, - mask, &vec_masks, - mask_vectype); - } + vect_get_slp_defs (op_node, gvec_oprnds[0]); + if (mask_node) + vect_get_slp_defs (mask_node, &vec_masks); if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) - vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info, - slp_node, &gs_info, + vect_get_gather_scatter_ops (loop, slp_node, &dataref_ptr, &vec_offsets); else dataref_ptr @@ -9455,13 +8873,13 @@ vectorizable_store (vec_info *vinfo, if (!costing_p) { vec_oprnd = (*gvec_oprnds[0])[j]; - if (mask) + if (mask_node) vec_mask = vec_masks[j]; /* We should have catched mismatched types earlier. */ - gcc_assert (useless_type_conversion_p (vectype, - TREE_TYPE (vec_oprnd))); + gcc_assert (ls.ls_type + || useless_type_conversion_p + (vectype, TREE_TYPE (vec_oprnd))); } - unsigned HOST_WIDE_INT align; tree final_mask = NULL_TREE; tree final_len = NULL_TREE; tree bias = NULL_TREE; @@ -9476,24 +8894,58 @@ vectorizable_store (vec_info *vinfo, final_mask, vec_mask, gsi); } - if (gs_info.ifn != IFN_LAST) + unsigned align = get_object_alignment (DR_REF (first_dr_info->dr)); + tree alias_align_ptr = build_int_cst (ref_type, align); + if (memory_access_type == VMAT_GATHER_SCATTER_IFN) { if (costing_p) { + if (ls.supported_offset_vectype) + inside_cost + += record_stmt_cost (cost_vec, 1, vector_stmt, + slp_node, 0, vect_body); + if (ls.supported_scale) + inside_cost + += record_stmt_cost (cost_vec, 1, vector_stmt, + slp_node, 0, vect_body); + unsigned int cnunits = vect_nunits_for_cost (vectype); inside_cost += record_stmt_cost (cost_vec, cnunits, scalar_store, - stmt_info, slp_node, 0, - vect_body); + slp_node, 0, vect_body); continue; } if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) vec_offset = vec_offsets[j]; - tree scale = size_int (gs_info.scale); + tree scale = size_int (SLP_TREE_GS_SCALE (slp_node)); + bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset)); + + /* Perform the offset conversion and scaling if necessary. */ + if (!strided + && (ls.supported_offset_vectype || ls.supported_scale)) + { + gimple_seq stmts = NULL; + if (ls.supported_offset_vectype) + vec_offset = gimple_convert + (&stmts, ls.supported_offset_vectype, vec_offset); + if (ls.supported_scale) + { + tree mult_cst = build_int_cst + (TREE_TYPE (TREE_TYPE (vec_offset)), + SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale); + tree mult = build_vector_from_val + (TREE_TYPE (vec_offset), mult_cst); + vec_offset = gimple_build + (&stmts, MULT_EXPR, TREE_TYPE (vec_offset), + vec_offset, mult); + scale = size_int (ls.supported_scale); + } + gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT); + } - if (gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE) + if (ls.gs.ifn == IFN_MASK_LEN_SCATTER_STORE) { if (loop_lens) final_len = vect_get_loop_len (loop_vinfo, gsi, @@ -9512,35 +8964,54 @@ vectorizable_store (vec_info *vinfo, } } + if (ls.ls_type) + { + gimple *conv_stmt + = gimple_build_assign (make_ssa_name (vectype), + VIEW_CONVERT_EXPR, + build1 (VIEW_CONVERT_EXPR, vectype, + vec_oprnd)); + vect_finish_stmt_generation (vinfo, stmt_info, conv_stmt, + gsi); + vec_oprnd = gimple_get_lhs (conv_stmt); + } + gcall *call; if (final_len && final_mask) { if (VECTOR_TYPE_P (TREE_TYPE (vec_offset))) call = gimple_build_call_internal ( - IFN_MASK_LEN_SCATTER_STORE, 7, dataref_ptr, + IFN_MASK_LEN_SCATTER_STORE, 8, dataref_ptr, + alias_align_ptr, vec_offset, scale, vec_oprnd, final_mask, final_len, bias); else /* Non-vector offset indicates that prefer to take MASK_LEN_STRIDED_STORE instead of the - IFN_MASK_SCATTER_STORE with direct stride arg. */ + IFN_MASK_SCATTER_STORE with direct stride arg. + Similar to the gather case we have checked the + alignment for a scatter already and assume + that the strided store has the same requirements. */ call = gimple_build_call_internal ( IFN_MASK_LEN_STRIDED_STORE, 6, dataref_ptr, vec_offset, vec_oprnd, final_mask, final_len, bias); } else if (final_mask) call = gimple_build_call_internal - (IFN_MASK_SCATTER_STORE, 5, dataref_ptr, + (IFN_MASK_SCATTER_STORE, 6, dataref_ptr, + alias_align_ptr, vec_offset, scale, vec_oprnd, final_mask); else - call = gimple_build_call_internal (IFN_SCATTER_STORE, 4, - dataref_ptr, vec_offset, + call = gimple_build_call_internal (IFN_SCATTER_STORE, 5, + dataref_ptr, + alias_align_ptr, + vec_offset, scale, vec_oprnd); gimple_call_set_nothrow (call, true); vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); new_stmt = call; } - else if (gs_info.decl) + else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY) { /* The builtin decls path for scatter is legacy, x86 only. */ gcc_assert (nunits.is_constant () @@ -9552,17 +9023,18 @@ vectorizable_store (vec_info *vinfo, unsigned int cnunits = vect_nunits_for_cost (vectype); inside_cost += record_stmt_cost (cost_vec, cnunits, scalar_store, - stmt_info, slp_node, 0, vect_body); + slp_node, 0, vect_body); continue; } + tree offset_vectype = TREE_TYPE (vec_offsets[0]); poly_uint64 offset_nunits - = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype); + = TYPE_VECTOR_SUBPARTS (offset_vectype); if (known_eq (nunits, offset_nunits)) { new_stmt = vect_build_one_scatter_store_call - (vinfo, stmt_info, gsi, &gs_info, - dataref_ptr, vec_offsets[j], + (vinfo, stmt_info, slp_node, gsi, + ls.gs.decl, dataref_ptr, vec_offsets[j], vec_oprnd, final_mask); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); @@ -9573,7 +9045,7 @@ vectorizable_store (vec_info *vinfo, lanes but the builtins will store full vectype data from the lower lanes. */ new_stmt = vect_build_one_scatter_store_call - (vinfo, stmt_info, gsi, &gs_info, + (vinfo, stmt_info, slp_node, gsi, ls.gs.decl, dataref_ptr, vec_offsets[2 * j], vec_oprnd, final_mask); vect_finish_stmt_generation (vinfo, stmt_info, @@ -9599,14 +9071,14 @@ vectorizable_store (vec_info *vinfo, VEC_UNPACK_HI_EXPR, final_mask); final_mask = make_ssa_name - (truth_type_for (gs_info.offset_vectype)); + (truth_type_for (offset_vectype)); gimple_set_lhs (new_stmt, final_mask); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } new_stmt = vect_build_one_scatter_store_call - (vinfo, stmt_info, gsi, &gs_info, + (vinfo, stmt_info, slp_node, gsi, ls.gs.decl, dataref_ptr, vec_offsets[2 * j + 1], vec_oprnd, final_mask); vect_finish_stmt_generation (vinfo, stmt_info, @@ -9639,8 +9111,8 @@ vectorizable_store (vec_info *vinfo, } new_stmt = vect_build_one_scatter_store_call - (vinfo, stmt_info, gsi, &gs_info, - dataref_ptr, vec_offset, + (vinfo, stmt_info, slp_node, gsi, + ls.gs.decl, dataref_ptr, vec_offset, vec_oprnd, final_mask); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); @@ -9660,20 +9132,21 @@ vectorizable_store (vec_info *vinfo, consumed by the load). */ inside_cost += record_stmt_cost (cost_vec, cnunits, vec_to_scalar, - stmt_info, slp_node, 0, vect_body); + slp_node, 0, vect_body); /* N scalar stores plus extracting the elements. */ inside_cost += record_stmt_cost (cost_vec, cnunits, vec_to_scalar, - stmt_info, slp_node, 0, vect_body); + slp_node, 0, vect_body); inside_cost += record_stmt_cost (cost_vec, cnunits, scalar_store, - stmt_info, slp_node, 0, vect_body); + slp_node, 0, vect_body); continue; } + tree offset_vectype = TREE_TYPE (vec_offsets[0]); unsigned HOST_WIDE_INT const_nunits = nunits.to_constant (); unsigned HOST_WIDE_INT const_offset_nunits - = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype).to_constant (); + = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant (); vec<constructor_elt, va_gc> *ctor_elts; vec_alloc (ctor_elts, const_nunits); gimple_seq stmts = NULL; @@ -9688,8 +9161,7 @@ vectorizable_store (vec_info *vinfo, unsigned elt_offset = (j % factor) * const_nunits; tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset)); - tree scale = size_int (gs_info.scale); - align = get_object_alignment (DR_REF (first_dr_info->dr)); + tree scale = size_int (SLP_TREE_GS_SCALE (slp_node)); tree ltype = build_aligned_type (TREE_TYPE (vectype), align); for (unsigned k = 0; k < const_nunits; ++k) { @@ -9722,17 +9194,10 @@ vectorizable_store (vec_info *vinfo, vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } - if (slp) - slp_node->push_vec_def (new_stmt); + slp_node->push_vec_def (new_stmt); } - - if (!slp && !costing_p) - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } - if (!slp && !costing_p) - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - if (costing_p && dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "vect_model_store_cost: inside_cost = %d, " @@ -9744,7 +9209,6 @@ vectorizable_store (vec_info *vinfo, gcc_assert (memory_access_type == VMAT_CONTIGUOUS || memory_access_type == VMAT_CONTIGUOUS_DOWN - || memory_access_type == VMAT_CONTIGUOUS_PERMUTE || memory_access_type == VMAT_CONTIGUOUS_REVERSE); unsigned inside_cost = 0, prologue_cost = 0; @@ -9753,332 +9217,211 @@ vectorizable_store (vec_info *vinfo, unsigned int n_adjacent_stores = 0; auto_vec<tree> result_chain (group_size); auto_vec<tree, 1> vec_oprnds; - for (j = 0; j < ncopies; j++) + gimple *new_stmt; + if (!costing_p) + { + /* Get vectorized arguments for SLP_NODE. */ + vect_get_slp_defs (op_node, &vec_oprnds); + vec_oprnd = vec_oprnds[0]; + if (mask_node) + { + vect_get_slp_defs (mask_node, &vec_masks); + vec_mask = vec_masks[0]; + } + } + + /* We should have catched mismatched types earlier. */ + gcc_assert (costing_p + || useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd))); + bool simd_lane_access_p + = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0; + if (!costing_p + && simd_lane_access_p + && !loop_masks + && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR + && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0)) + && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info)) + && integer_zerop (DR_INIT (first_dr_info->dr)) + && alias_sets_conflict_p (get_alias_set (aggr_type), + get_alias_set (TREE_TYPE (ref_type)))) + { + dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr)); + dataref_offset = build_int_cst (ref_type, 0); + } + else if (!costing_p) + dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type, + simd_lane_access_p ? loop : NULL, + offset, &dummy, gsi, &ptr_incr, + simd_lane_access_p, bump); + + new_stmt = NULL; + gcc_assert (!grouped_store); + for (i = 0; i < vec_num; i++) { - gimple *new_stmt; - if (j == 0) + if (!costing_p) + vec_oprnd = vec_oprnds[i]; + + if (memory_access_type == VMAT_CONTIGUOUS_REVERSE) { - if (slp && !costing_p) - { - /* Get vectorized arguments for SLP_NODE. */ - vect_get_vec_defs (vinfo, stmt_info, slp_node, 1, op, - &vec_oprnds, mask, &vec_masks); - vec_oprnd = vec_oprnds[0]; - if (mask) - vec_mask = vec_masks[0]; - } + if (costing_p) + inside_cost += record_stmt_cost (cost_vec, 1, vec_perm, + slp_node, 0, vect_body); else { - /* For interleaved stores we collect vectorized defs for all the - stores in the group in DR_CHAIN. DR_CHAIN is then used as an - input to vect_permute_store_chain(). + tree perm_mask = perm_mask_for_reverse (vectype); + tree new_temp = make_ssa_name (vectype); - If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN - is of size 1. */ - stmt_vec_info next_stmt_info = first_stmt_info; - for (i = 0; i < group_size; i++) - { - /* Since gaps are not supported for interleaved stores, - DR_GROUP_SIZE is the exact number of stmts in the chain. - Therefore, NEXT_STMT_INFO can't be NULL_TREE. In case - that there is no interleaving, DR_GROUP_SIZE is 1, - and only one iteration of the loop will be executed. */ - op = vect_get_store_rhs (next_stmt_info); - if (costing_p) - update_prologue_cost (&prologue_cost, op); - else - { - vect_get_vec_defs_for_operand (vinfo, next_stmt_info, - ncopies, op, - gvec_oprnds[i]); - vec_oprnd = (*gvec_oprnds[i])[0]; - dr_chain.quick_push (vec_oprnd); - } - next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info); - } - if (mask && !costing_p) - { - vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies, - mask, &vec_masks, - mask_vectype); - vec_mask = vec_masks[0]; - } - } + /* Generate the permute statement. */ + gimple *perm_stmt + = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd, + vec_oprnd, perm_mask); + vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi); - /* We should have catched mismatched types earlier. */ - gcc_assert (costing_p - || useless_type_conversion_p (vectype, - TREE_TYPE (vec_oprnd))); - bool simd_lane_access_p - = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0; - if (!costing_p - && simd_lane_access_p - && !loop_masks - && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR - && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0)) - && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info)) - && integer_zerop (DR_INIT (first_dr_info->dr)) - && alias_sets_conflict_p (get_alias_set (aggr_type), - get_alias_set (TREE_TYPE (ref_type)))) - { - dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr)); - dataref_offset = build_int_cst (ref_type, 0); + perm_stmt = SSA_NAME_DEF_STMT (new_temp); + vec_oprnd = new_temp; } - else if (!costing_p) - dataref_ptr - = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type, - simd_lane_access_p ? loop : NULL, - offset, &dummy, gsi, &ptr_incr, - simd_lane_access_p, bump); } - else if (!costing_p) + + if (costing_p) { - gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)); - /* DR_CHAIN is then used as an input to vect_permute_store_chain(). - If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is - of size 1. */ - for (i = 0; i < group_size; i++) - { - vec_oprnd = (*gvec_oprnds[i])[j]; - dr_chain[i] = vec_oprnd; - } - if (mask) - vec_mask = vec_masks[j]; - if (dataref_offset) - dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump); - else - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, - stmt_info, bump); + n_adjacent_stores++; + continue; } - new_stmt = NULL; - if (grouped_store) + tree final_mask = NULL_TREE; + tree final_len = NULL_TREE; + tree bias = NULL_TREE; + if (loop_masks) + final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks, + vec_num, vectype, i); + if (vec_mask) + vec_mask = vec_masks[i]; + if (vec_mask) + final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask, + vec_mask, gsi); + + if (i > 0) + /* Bump the vector pointer. */ + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, + stmt_info, bump); + + unsigned misalign; + unsigned HOST_WIDE_INT align; + align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info)); + if (alignment_support_scheme == dr_aligned) + misalign = 0; + else if (misalignment == DR_MISALIGNMENT_UNKNOWN) { - /* Permute. */ - gcc_assert (memory_access_type == VMAT_CONTIGUOUS_PERMUTE); - if (costing_p) - { - int group_size = DR_GROUP_SIZE (first_stmt_info); - int nstmts = ceil_log2 (group_size) * group_size; - inside_cost += record_stmt_cost (cost_vec, nstmts, vec_perm, - stmt_info, slp_node, 0, - vect_body); - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "vect_model_store_cost: " - "strided group_size = %d .\n", - group_size); - } - else - vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info, - gsi, &result_chain); + align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info)); + misalign = 0; + } + else + misalign = misalignment; + if (dataref_offset == NULL_TREE + && TREE_CODE (dataref_ptr) == SSA_NAME) + set_ptr_info_alignment (get_ptr_info (dataref_ptr), align, misalign); + align = least_bit_hwi (misalign | align); + + /* Compute IFN when LOOP_LENS or final_mask valid. */ + machine_mode vmode = TYPE_MODE (vectype); + machine_mode new_vmode = vmode; + internal_fn partial_ifn = IFN_LAST; + if (loop_lens) + { + opt_machine_mode new_ovmode + = get_len_load_store_mode (vmode, false, &partial_ifn); + new_vmode = new_ovmode.require (); + unsigned factor + = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode); + final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens, + vec_num, vectype, i, factor); + } + else if (final_mask) + { + if (!can_vec_mask_load_store_p (vmode, + TYPE_MODE (TREE_TYPE (final_mask)), + false, &partial_ifn)) + gcc_unreachable (); } - stmt_vec_info next_stmt_info = first_stmt_info; - for (i = 0; i < vec_num; i++) + if (partial_ifn == IFN_MASK_LEN_STORE) { - if (!costing_p) + if (!final_len) { - if (slp) - vec_oprnd = vec_oprnds[i]; - else if (grouped_store) - /* For grouped stores vectorized defs are interleaved in - vect_permute_store_chain(). */ - vec_oprnd = result_chain[i]; + /* Pass VF value to 'len' argument of + MASK_LEN_STORE if LOOP_LENS is invalid. */ + final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype)); } - - if (memory_access_type == VMAT_CONTIGUOUS_REVERSE) + if (!final_mask) { - if (costing_p) - inside_cost += record_stmt_cost (cost_vec, 1, vec_perm, - stmt_info, slp_node, 0, - vect_body); - else - { - tree perm_mask = perm_mask_for_reverse (vectype); - tree perm_dest = vect_create_destination_var ( - vect_get_store_rhs (stmt_info), vectype); - tree new_temp = make_ssa_name (perm_dest); - - /* Generate the permute statement. */ - gimple *perm_stmt - = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd, - vec_oprnd, perm_mask); - vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, - gsi); - - perm_stmt = SSA_NAME_DEF_STMT (new_temp); - vec_oprnd = new_temp; - } - } - - if (costing_p) - { - n_adjacent_stores++; - - if (!slp) - { - next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info); - if (!next_stmt_info) - break; - } - - continue; + /* Pass all ones value to 'mask' argument of + MASK_LEN_STORE if final_mask is invalid. */ + mask_vectype = truth_type_for (vectype); + final_mask = build_minus_one_cst (mask_vectype); } + } + if (final_len) + { + signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); + bias = build_int_cst (intQI_type_node, biasval); + } - tree final_mask = NULL_TREE; - tree final_len = NULL_TREE; - tree bias = NULL_TREE; - if (loop_masks) - final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks, - vec_num * ncopies, vectype, - vec_num * j + i); - if (slp && vec_mask) - vec_mask = vec_masks[i]; - if (vec_mask) - final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask, - vec_mask, gsi); - - if (i > 0) - /* Bump the vector pointer. */ - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, - stmt_info, bump); - - unsigned misalign; - unsigned HOST_WIDE_INT align; - align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info)); - if (alignment_support_scheme == dr_aligned) - misalign = 0; - else if (misalignment == DR_MISALIGNMENT_UNKNOWN) - { - align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info)); - misalign = 0; - } - else - misalign = misalignment; - if (dataref_offset == NULL_TREE - && TREE_CODE (dataref_ptr) == SSA_NAME) - set_ptr_info_alignment (get_ptr_info (dataref_ptr), align, - misalign); - align = least_bit_hwi (misalign | align); - - /* Compute IFN when LOOP_LENS or final_mask valid. */ - machine_mode vmode = TYPE_MODE (vectype); - machine_mode new_vmode = vmode; - internal_fn partial_ifn = IFN_LAST; - if (loop_lens) - { - opt_machine_mode new_ovmode - = get_len_load_store_mode (vmode, false, &partial_ifn); - new_vmode = new_ovmode.require (); - unsigned factor - = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode); - final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens, - vec_num * ncopies, vectype, - vec_num * j + i, factor); - } - else if (final_mask) - { - if (!can_vec_mask_load_store_p ( - vmode, TYPE_MODE (TREE_TYPE (final_mask)), false, - &partial_ifn)) - gcc_unreachable (); + /* Arguments are ready. Create the new vector stmt. */ + if (final_len) + { + gcall *call; + tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT); + /* Need conversion if it's wrapped with VnQI. */ + if (vmode != new_vmode) + { + tree new_vtype + = build_vector_type_for_mode (unsigned_intQI_type_node, + new_vmode); + tree var = vect_get_new_ssa_name (new_vtype, vect_simple_var); + vec_oprnd = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd); + gassign *new_stmt + = gimple_build_assign (var, VIEW_CONVERT_EXPR, vec_oprnd); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vec_oprnd = var; } if (partial_ifn == IFN_MASK_LEN_STORE) - { - if (!final_len) - { - /* Pass VF value to 'len' argument of - MASK_LEN_STORE if LOOP_LENS is invalid. */ - final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype)); - } - if (!final_mask) - { - /* Pass all ones value to 'mask' argument of - MASK_LEN_STORE if final_mask is invalid. */ - mask_vectype = truth_type_for (vectype); - final_mask = build_minus_one_cst (mask_vectype); - } - } - if (final_len) - { - signed char biasval - = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); - - bias = build_int_cst (intQI_type_node, biasval); - } - - /* Arguments are ready. Create the new vector stmt. */ - if (final_len) - { - gcall *call; - tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT); - /* Need conversion if it's wrapped with VnQI. */ - if (vmode != new_vmode) - { - tree new_vtype - = build_vector_type_for_mode (unsigned_intQI_type_node, - new_vmode); - tree var = vect_get_new_ssa_name (new_vtype, vect_simple_var); - vec_oprnd = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd); - gassign *new_stmt - = gimple_build_assign (var, VIEW_CONVERT_EXPR, vec_oprnd); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - vec_oprnd = var; - } - - if (partial_ifn == IFN_MASK_LEN_STORE) - call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6, - dataref_ptr, ptr, final_mask, - final_len, bias, vec_oprnd); - else - call = gimple_build_call_internal (IFN_LEN_STORE, 5, - dataref_ptr, ptr, final_len, - bias, vec_oprnd); - gimple_call_set_nothrow (call, true); - vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); - new_stmt = call; - } - else if (final_mask) - { - tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT); - gcall *call - = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr, - ptr, final_mask, vec_oprnd); - gimple_call_set_nothrow (call, true); - vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); - new_stmt = call; - } + call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6, + dataref_ptr, ptr, final_mask, + final_len, bias, vec_oprnd); else - { - data_ref - = fold_build2 (MEM_REF, vectype, dataref_ptr, - dataref_offset ? dataref_offset - : build_int_cst (ref_type, 0)); - if (alignment_support_scheme == dr_aligned) - ; - else - TREE_TYPE (data_ref) - = build_aligned_type (TREE_TYPE (data_ref), - align * BITS_PER_UNIT); - vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); - new_stmt = gimple_build_assign (data_ref, vec_oprnd); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - } - - if (slp) - continue; - - next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info); - if (!next_stmt_info) - break; + call = gimple_build_call_internal (IFN_LEN_STORE, 5, + dataref_ptr, ptr, final_len, + bias, vec_oprnd); + gimple_call_set_nothrow (call, true); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; } - if (!slp && !costing_p) + else if (final_mask) { - if (j == 0) - *vec_stmt = new_stmt; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT); + gcall *call + = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr, + ptr, final_mask, vec_oprnd); + gimple_call_set_nothrow (call, true); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; + } + else + { + data_ref = fold_build2 (MEM_REF, vectype, dataref_ptr, + dataref_offset ? dataref_offset + : build_int_cst (ref_type, 0)); + if (alignment_support_scheme == dr_aligned + && align >= TYPE_ALIGN_UNIT (vectype)) + ; + else + TREE_TYPE (data_ref) + = build_aligned_type (TREE_TYPE (data_ref), + align * BITS_PER_UNIT); + vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); + new_stmt = gimple_build_assign (data_ref, vec_oprnd); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } } @@ -10110,12 +9453,12 @@ vectorizable_store (vec_info *vinfo, { /* Spill. */ prologue_cost - += record_stmt_cost (cost_vec, ncopies, vector_store, - stmt_info, slp_node, 0, vect_epilogue); + += record_stmt_cost (cost_vec, 1, vector_store, + slp_node, 0, vect_epilogue); /* Loads. */ prologue_cost - += record_stmt_cost (cost_vec, ncopies * nregs, scalar_load, - stmt_info, slp_node, 0, vect_epilogue); + += record_stmt_cost (cost_vec, nregs, scalar_load, + slp_node, 0, vect_epilogue); } } } @@ -10255,14 +9598,15 @@ hoist_defs_of_uses (gimple *stmt, class loop *loop, bool hoist_p) Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure) that can be vectorized. - If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized - stmt to replace it, put it in VEC_STMT, and insert it at GSI. + If COST_VEC is passed, calculate costs but don't change anything, + otherwise, vectorize STMT_INFO: create a vectorized stmt to replace + it, and insert it at GSI. Return true if STMT_INFO is vectorizable in this way. */ static bool vectorizable_load (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - gimple **vec_stmt, slp_tree slp_node, + slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree scalar_dest; @@ -10280,26 +9624,21 @@ vectorizable_load (vec_info *vinfo, tree dataref_ptr = NULL_TREE; tree dataref_offset = NULL_TREE; gimple *ptr_incr = NULL; - int ncopies; int i, j; unsigned int group_size; poly_uint64 group_gap_adj; tree msq = NULL_TREE, lsq; tree realignment_token = NULL_TREE; gphi *phi = NULL; - vec<tree> dr_chain = vNULL; bool grouped_load = false; stmt_vec_info first_stmt_info; stmt_vec_info first_stmt_info_for_drptr = NULL; bool compute_in_loop = false; class loop *at_loop; int vec_num; - bool slp = (slp_node != NULL); - bool slp_perm = false; bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo); poly_uint64 vf; tree aggr_type; - gather_scatter_info gs_info; tree ref_type; enum vect_def_type mask_dt = vect_unknown_def_type; enum vect_def_type els_dt = vect_unknown_def_type; @@ -10308,18 +9647,18 @@ vectorizable_load (vec_info *vinfo, return false; if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def - && ! vec_stmt) + && cost_vec) return false; if (!STMT_VINFO_DATA_REF (stmt_info)) return false; - tree mask = NULL_TREE, mask_vectype = NULL_TREE; + tree mask_vectype = NULL_TREE; tree els = NULL_TREE; tree els_vectype = NULL_TREE; int mask_index = -1; int els_index = -1; - slp_tree slp_op = NULL; + slp_tree mask_node = NULL; slp_tree els_op = NULL; if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt)) { @@ -10353,25 +9692,25 @@ vectorizable_load (vec_info *vinfo, return false; mask_index = internal_fn_mask_index (ifn); - if (mask_index >= 0 && slp_node) + if (mask_index >= 0) mask_index = vect_slp_child_index_for_operand (call, mask_index, STMT_VINFO_GATHER_SCATTER_P (stmt_info)); if (mask_index >= 0 - && !vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_index, - &mask, &slp_op, &mask_dt, &mask_vectype)) + && !vect_check_scalar_mask (vinfo, slp_node, mask_index, + &mask_node, &mask_dt, &mask_vectype)) return false; els_index = internal_fn_else_index (ifn); - if (els_index >= 0 && slp_node) + if (els_index >= 0) els_index = vect_slp_child_index_for_operand (call, els_index, STMT_VINFO_GATHER_SCATTER_P (stmt_info)); if (els_index >= 0 - && !vect_is_simple_use (vinfo, stmt_info, slp_node, els_index, + && !vect_is_simple_use (vinfo, slp_node, els_index, &els, &els_op, &els_dt, &els_vectype)) return false; } - tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vectype = SLP_TREE_VECTYPE (slp_node); poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); if (loop_vinfo) @@ -10383,19 +9722,10 @@ vectorizable_load (vec_info *vinfo, else vf = 1; - /* Multiple types in SLP are handled by creating the appropriate number of - vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in - case of SLP. */ - if (slp) - ncopies = 1; - else - ncopies = vect_get_num_copies (loop_vinfo, vectype); - - gcc_assert (ncopies >= 1); + vec_num = vect_get_num_copies (vinfo, slp_node); /* FORNOW. This restriction should be relaxed. */ - if (nested_in_vect_loop - && (ncopies > 1 || (slp && SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) > 1))) + if (nested_in_vect_loop && vec_num > 1) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -10403,20 +9733,6 @@ vectorizable_load (vec_info *vinfo, return false; } - /* Invalidate assumptions made by dependence analysis when vectorization - on the unrolled body effectively re-orders stmts. */ - if (ncopies > 1 - && STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0 - && maybe_gt (LOOP_VINFO_VECT_FACTOR (loop_vinfo), - STMT_VINFO_MIN_NEG_DIST (stmt_info))) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "cannot perform implicit CSE when unrolling " - "with negative dependence distance\n"); - return false; - } - elem_type = TREE_TYPE (vectype); mode = TYPE_MODE (vectype); @@ -10441,15 +9757,6 @@ vectorizable_load (vec_info *vinfo, first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info); group_size = DR_GROUP_SIZE (first_stmt_info); - /* Refuse non-SLP vectorization of SLP-only groups. */ - if (!slp && STMT_VINFO_SLP_VECT_ONLY (first_stmt_info)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "cannot vectorize load in non-SLP mode.\n"); - return false; - } - /* Invalidate assumptions made by dependence analysis when vectorization on the unrolled body effectively re-orders stmts. */ if (STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0 @@ -10466,20 +9773,23 @@ vectorizable_load (vec_info *vinfo, else group_size = 1; - vect_memory_access_type memory_access_type; - enum dr_alignment_support alignment_support_scheme; - int misalignment; - poly_int64 poffset; - internal_fn lanes_ifn; - auto_vec<int> elsvals; - int maskload_elsval = 0; - bool need_zeroing = false; - if (!get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask, VLS_LOAD, - ncopies, &memory_access_type, &poffset, - &alignment_support_scheme, &misalignment, &gs_info, - &lanes_ifn, &elsvals)) + vect_load_store_data _ls_data{}; + vect_load_store_data &ls = slp_node->get_data (_ls_data); + if (cost_vec + && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node, + VLS_LOAD, &ls)) return false; + /* Temporary aliases to analysis data, should not be modified through + these. */ + const vect_memory_access_type memory_access_type = ls.memory_access_type; + const dr_alignment_support alignment_support_scheme + = ls.alignment_support_scheme; + const int misalignment = ls.misalignment; + const poly_int64 poffset = ls.poffset; + const vec<int> &elsvals = ls.elsvals; + int maskload_elsval = 0; + bool need_zeroing = false; /* We might need to explicitly zero inactive elements if there are padding bits in the type that might leak otherwise. @@ -10488,54 +9798,7 @@ vectorizable_load (vec_info *vinfo, bool type_mode_padding_p = TYPE_PRECISION (scalar_type) < GET_MODE_PRECISION (GET_MODE_INNER (mode)); - /* ??? The following checks should really be part of - get_group_load_store_type. */ - if (slp - && SLP_TREE_LOAD_PERMUTATION (slp_node).exists () - && !((memory_access_type == VMAT_ELEMENTWISE - || memory_access_type == VMAT_GATHER_SCATTER) - && SLP_TREE_LANES (slp_node) == 1)) - { - slp_perm = true; - - if (!loop_vinfo) - { - /* In BB vectorization we may not actually use a loaded vector - accessing elements in excess of DR_GROUP_SIZE. */ - stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (slp_node)[0]; - group_info = DR_GROUP_FIRST_ELEMENT (group_info); - unsigned HOST_WIDE_INT nunits; - unsigned j, k, maxk = 0; - FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (slp_node), j, k) - if (k > maxk) - maxk = k; - tree vectype = SLP_TREE_VECTYPE (slp_node); - if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits) - || maxk >= (DR_GROUP_SIZE (group_info) & ~(nunits - 1))) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "BB vectorization with gaps at the end of " - "a load is not supported\n"); - return false; - } - } - - auto_vec<tree> tem; - unsigned n_perms; - if (!vect_transform_slp_perm_load (vinfo, slp_node, tem, NULL, vf, - true, &n_perms)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, - vect_location, - "unsupported load permutation\n"); - return false; - } - } - - if (slp_node - && slp_node->ldst_lanes + if (slp_node->ldst_lanes && memory_access_type != VMAT_LOAD_STORE_LANES) { if (dump_enabled_p ()) @@ -10544,7 +9807,7 @@ vectorizable_load (vec_info *vinfo, return false; } - if (mask) + if (mask_node) { if (memory_access_type == VMAT_CONTIGUOUS) { @@ -10552,20 +9815,18 @@ vectorizable_load (vec_info *vinfo, if (!VECTOR_MODE_P (vec_mode) || !can_vec_mask_load_store_p (vec_mode, TYPE_MODE (mask_vectype), - true, NULL, &elsvals)) + true, NULL, &ls.elsvals)) return false; } else if (memory_access_type != VMAT_LOAD_STORE_LANES - && memory_access_type != VMAT_GATHER_SCATTER) + && !mat_gather_scatter_p (memory_access_type)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, "unsupported access type for masked load.\n"); return false; } - else if (memory_access_type == VMAT_GATHER_SCATTER - && gs_info.ifn == IFN_LAST - && !gs_info.decl) + else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -10582,13 +9843,12 @@ vectorizable_load (vec_info *vinfo, } } - bool costing_p = !vec_stmt; + bool costing_p = cost_vec; if (costing_p) /* transformation not required. */ { - if (slp_node - && mask - && !vect_maybe_update_slp_op_vectype (slp_op, + if (mask_node + && !vect_maybe_update_slp_op_vectype (mask_node, mask_vectype)) { if (dump_enabled_p ()) @@ -10597,21 +9857,15 @@ vectorizable_load (vec_info *vinfo, return false; } - if (!slp) - STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type; - else - SLP_TREE_MEMORY_ACCESS_TYPE (slp_node) = memory_access_type; - if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)) check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node, - VLS_LOAD, group_size, - memory_access_type, &gs_info, - mask, &elsvals); + VLS_LOAD, group_size, &ls, + mask_node, &ls.elsvals); if (dump_enabled_p () && memory_access_type != VMAT_ELEMENTWISE - && memory_access_type != VMAT_GATHER_SCATTER + && !mat_gather_scatter_p (memory_access_type) && memory_access_type != VMAT_STRIDED_SLP && memory_access_type != VMAT_INVARIANT && alignment_support_scheme != dr_aligned) @@ -10621,17 +9875,8 @@ vectorizable_load (vec_info *vinfo, if (memory_access_type == VMAT_LOAD_STORE_LANES) vinfo->any_known_not_updated_vssa = true; - STMT_VINFO_TYPE (stmt_info) = load_vec_info_type; - } - else - { - /* Here just get the else values. */ - if (loop_vinfo - && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)) - check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node, - VLS_LOAD, group_size, - memory_access_type, &gs_info, - mask, &elsvals); + SLP_TREE_TYPE (slp_node) = load_vec_info_type; + slp_node->data = new vect_load_store_data (std::move (ls)); } /* If the type needs padding we must zero inactive elements. @@ -10654,16 +9899,8 @@ vectorizable_load (vec_info *vinfo, if (elsvals.length ()) maskload_elsval = *elsvals.begin (); - if (!slp) - gcc_assert (memory_access_type - == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info)); - else - gcc_assert (memory_access_type - == SLP_TREE_MEMORY_ACCESS_TYPE (slp_node)); - if (dump_enabled_p () && !costing_p) - dump_printf_loc (MSG_NOTE, vect_location, - "transform load. ncopies = %d\n", ncopies); + dump_printf_loc (MSG_NOTE, vect_location, "transform load.\n"); /* Transform. */ @@ -10672,7 +9909,7 @@ vectorizable_load (vec_info *vinfo, if (memory_access_type == VMAT_INVARIANT) { - gcc_assert (!grouped_load && !mask && !bb_vinfo); + gcc_assert (!grouped_load && !mask_node && !bb_vinfo); /* If we have versioned for aliasing or the loop doesn't have any data dependencies that would preclude this, then we are sure this is a loop invariant load and @@ -10681,16 +9918,56 @@ vectorizable_load (vec_info *vinfo, once at analysis time, remembered and used in the transform time. */ bool hoist_p = (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) - && !nested_in_vect_loop - && hoist_defs_of_uses (stmt_info->stmt, loop, false)); + && !nested_in_vect_loop); + + bool uniform_p = true; + for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node)) + { + /* It is unsafe to hoist a conditional load over the conditions that + make it valid. When early break this means that any invariant load + can't be hoisted unless it's in the loop header or if we know + something else has verified the load is valid to do. Alignment + peeling would do this since getting through the prologue means the + load was done at least once and so the vector main body is free to + hoist it. However today GCC will hoist the load above the PFA + loop. As such that makes it still invalid and so we can't allow it + today. */ + if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo) + && !DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (sinfo)) + && gimple_bb (STMT_VINFO_STMT (vect_orig_stmt (sinfo))) + != loop->header) + { + if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) + && dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "not hoisting invariant load due to early break" + "constraints\n"); + else if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "not hoisting invariant load due to early break" + "constraints\n"); + hoist_p = false; + } + + hoist_p = hoist_p && hoist_defs_of_uses (sinfo->stmt, loop, false); + if (sinfo != SLP_TREE_SCALAR_STMTS (slp_node)[0]) + uniform_p = false; + } if (costing_p) { + if (!uniform_p && (!hoist_p || !vf.is_constant ())) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "not vectorizing non-uniform invariant " + "load\n"); + return false; + } enum vect_cost_model_location cost_loc = hoist_p ? vect_prologue : vect_body; unsigned int cost = record_stmt_cost (cost_vec, 1, scalar_load, - stmt_info, slp_node, 0, - cost_loc); - cost += record_stmt_cost (cost_vec, 1, scalar_to_vec, stmt_info, + slp_node, 0, cost_loc); + cost += record_stmt_cost (cost_vec, 1, scalar_to_vec, slp_node, 0, cost_loc); unsigned int prologue_cost = hoist_p ? cost : 0; unsigned int inside_cost = hoist_p ? 0 : cost; @@ -10703,46 +9980,77 @@ vectorizable_load (vec_info *vinfo, } if (hoist_p) { - gassign *stmt = as_a <gassign *> (stmt_info->stmt); - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "hoisting out of the vectorized loop: %G", - (gimple *) stmt); - scalar_dest = copy_ssa_name (scalar_dest); - tree rhs = unshare_expr (gimple_assign_rhs1 (stmt)); - edge pe = loop_preheader_edge (loop); - gphi *vphi = get_virtual_phi (loop->header); - tree vuse; - if (vphi) - vuse = PHI_ARG_DEF_FROM_EDGE (vphi, pe); - else - vuse = gimple_vuse (gsi_stmt (*gsi)); - gimple *new_stmt = gimple_build_assign (scalar_dest, rhs); - gimple_set_vuse (new_stmt, vuse); - gsi_insert_on_edge_immediate (pe, new_stmt); - hoist_defs_of_uses (new_stmt, loop, true); + /* ??? For non-uniform lanes there could be still duplicates. + We're leaving those to post-vectorizer CSE for the moment. */ + auto_vec<tree> scalar_defs (SLP_TREE_LANES (slp_node)); + for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node)) + { + gassign *stmt = as_a <gassign *> (sinfo->stmt); + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "hoisting out of the vectorized loop: %G", + (gimple *) stmt); + scalar_dest = copy_ssa_name (gimple_assign_lhs (stmt)); + tree rhs = unshare_expr (gimple_assign_rhs1 (stmt)); + edge pe = loop_preheader_edge (loop); + gphi *vphi = get_virtual_phi (loop->header); + tree vuse; + if (vphi) + vuse = PHI_ARG_DEF_FROM_EDGE (vphi, pe); + else + vuse = gimple_vuse (gsi_stmt (*gsi)); + gimple *new_stmt = gimple_build_assign (scalar_dest, rhs); + gimple_set_vuse (new_stmt, vuse); + gsi_insert_on_edge_immediate (pe, new_stmt); + hoist_defs_of_uses (new_stmt, loop, true); + if (!useless_type_conversion_p (TREE_TYPE (vectype), + TREE_TYPE (scalar_dest))) + { + tree tem = make_ssa_name (TREE_TYPE (vectype)); + new_stmt = gimple_build_assign (tem, + NOP_EXPR, scalar_dest); + gsi_insert_on_edge_immediate (pe, new_stmt); + scalar_dest = tem; + } + scalar_defs.quick_push (scalar_dest); + if (uniform_p) + break; + } + if (!uniform_p) + { + unsigned const_nunits + = TYPE_VECTOR_SUBPARTS (vectype).to_constant (); + for (j = 0; j < (int) vec_num; ++j) + { + vec<constructor_elt, va_gc> *v = NULL; + vec_safe_reserve (v, const_nunits, true); + for (unsigned i = 0; i < const_nunits; ++i) + { + unsigned def_idx + = (j * const_nunits + i) % SLP_TREE_LANES (slp_node); + CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, + scalar_defs[def_idx]); + } + scalar_dest = build_constructor (vectype, v); + new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest, + vectype, NULL); + slp_node->push_vec_def (new_temp); + } + return true; + } + new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest, + vectype, NULL); } - /* These copies are all equivalent. */ - if (hoist_p) - new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest, - vectype, NULL); else { + gcc_assert (uniform_p); gimple_stmt_iterator gsi2 = *gsi; gsi_next (&gsi2); new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest, vectype, &gsi2); } - gimple *new_stmt = SSA_NAME_DEF_STMT (new_temp); - if (slp) - for (j = 0; j < (int) SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); ++j) - slp_node->push_vec_def (new_stmt); - else - { - for (j = 0; j < ncopies; ++j) - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); - *vec_stmt = new_stmt; - } + for (j = 0; j < (int) vec_num; ++j) + slp_node->push_vec_def (new_temp); return true; } @@ -10767,31 +10075,40 @@ vectorizable_load (vec_info *vinfo, if (grouped_load) { - first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info); + /* If we elided a consecutive load permutation, don't + use the original first statement (which could be elided) + but the one the load permutation starts with. + This ensures the stride_base below is correct. */ + if (!ls.subchain_p) + first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info); + else + first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0]; first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info); + ref_type = get_group_alias_ptr_type (first_stmt_info); } else { first_stmt_info = stmt_info; first_dr_info = dr_info; + ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)); } - if (slp && grouped_load - && memory_access_type == VMAT_STRIDED_SLP) + if (grouped_load) { - group_size = DR_GROUP_SIZE (first_stmt_info); - ref_type = get_group_alias_ptr_type (first_stmt_info); + if (memory_access_type == VMAT_STRIDED_SLP) + { + /* If we elided a consecutive load permutation, adjust + the group size here. */ + if (!ls.subchain_p) + group_size = DR_GROUP_SIZE (first_stmt_info); + else + group_size = SLP_TREE_LANES (slp_node); + } + else /* VMAT_ELEMENTWISE */ + group_size = SLP_TREE_LANES (slp_node); } else - { - if (grouped_load) - cst_offset - = (tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype))) - * vect_get_place_in_interleaving_chain (stmt_info, - first_stmt_info)); - group_size = 1; - ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)); - } + group_size = 1; if (!costing_p) { @@ -10839,6 +10156,13 @@ vectorizable_load (vec_info *vinfo, tree ltype = TREE_TYPE (vectype); tree lvectype = vectype; auto_vec<tree> dr_chain; + /* ??? Modify local copies of alignment_support_scheme and + misalignment, but this part of analysis should be done + earlier and remembered, likewise the chosen load mode. */ + const dr_alignment_support tem = alignment_support_scheme; + dr_alignment_support alignment_support_scheme = tem; + const int tem2 = misalignment; + int misalignment = tem2; if (memory_access_type == VMAT_STRIDED_SLP) { HOST_WIDE_INT n = gcd (group_size, const_nunits); @@ -10847,6 +10171,15 @@ vectorizable_load (vec_info *vinfo, if (n == const_nunits) { int mis_align = dr_misalignment (first_dr_info, vectype); + /* With VF > 1 we advance the DR by step, if that is constant + and only aligned when performed VF times, DR alignment + analysis can analyze this as aligned since it assumes + contiguous accesses. But that is not how we code generate + here, so adjust for this. */ + if (maybe_gt (vf, 1u) + && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr), + DR_TARGET_ALIGNMENT (first_dr_info))) + mis_align = -1; dr_alignment_support dr_align = vect_supportable_dr_alignment (vinfo, dr_info, vectype, mis_align); @@ -10875,6 +10208,10 @@ vectorizable_load (vec_info *vinfo, if (VECTOR_TYPE_P (ptype)) { mis_align = dr_misalignment (first_dr_info, ptype); + if (maybe_gt (vf, 1u) + && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr), + DR_TARGET_ALIGNMENT (first_dr_info))) + mis_align = -1; dr_align = vect_supportable_dr_alignment (vinfo, dr_info, ptype, mis_align); @@ -10894,30 +10231,32 @@ vectorizable_load (vec_info *vinfo, } } unsigned align; - /* ??? The above is still wrong, see vectorizable_store. */ - align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info)); + if (alignment_support_scheme == dr_aligned) + align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info)); + else + align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info)); /* Alignment is at most the access size if we do multiple loads. */ if (nloads > 1) align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align); ltype = build_aligned_type (ltype, align * BITS_PER_UNIT); } - if (slp) + /* For SLP permutation support we need to load the whole group, + not only the number of vector stmts the permutation result + fits in. */ + int ncopies; + if (ls.slp_perm) { - /* For SLP permutation support we need to load the whole group, - not only the number of vector stmts the permutation result - fits in. */ - if (slp_perm) - { - /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for - variable VF. */ - unsigned int const_vf = vf.to_constant (); - ncopies = CEIL (group_size * const_vf, const_nunits); - dr_chain.create (ncopies); - } - else - ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); + gcc_assert (memory_access_type != VMAT_ELEMENTWISE); + /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for + variable VF. */ + unsigned int const_vf = vf.to_constant (); + ncopies = CEIL (group_size * const_vf, const_nunits); + dr_chain.create (ncopies); } + else + ncopies = vec_num; + unsigned int group_el = 0; unsigned HOST_WIDE_INT elsz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype))); @@ -10941,12 +10280,16 @@ vectorizable_load (vec_info *vinfo, n_adjacent_loads++; else inside_cost += record_stmt_cost (cost_vec, 1, scalar_load, - stmt_info, slp_node, 0, - vect_body); + slp_node, 0, vect_body); continue; } + unsigned int load_el = group_el; + /* For elementwise accesses apply a load permutation directly. */ + if (memory_access_type == VMAT_ELEMENTWISE + && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) + load_el = SLP_TREE_LOAD_PERMUTATION (slp_node)[group_el]; tree this_off = build_int_cst (TREE_TYPE (alias_off), - group_el * elsz + cst_offset); + load_el * elsz + cst_offset); tree data_ref = build2 (MEM_REF, ltype, running_off, this_off); vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); new_temp = make_ssa_name (ltype); @@ -10956,14 +10299,13 @@ vectorizable_load (vec_info *vinfo, CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, new_temp); group_el += lnel; - if (! slp - || group_el == group_size) + if (group_el == group_size) { n_groups++; /* When doing SLP make sure to not load elements from the next vector iteration, those will not be accessed so just use the last element again. See PR107451. */ - if (!slp || known_lt (n_groups, vf)) + if (known_lt (n_groups, vf)) { tree newoff = copy_ssa_name (running_off); gimple *incr @@ -10980,8 +10322,7 @@ vectorizable_load (vec_info *vinfo, { if (costing_p) inside_cost += record_stmt_cost (cost_vec, 1, vec_construct, - stmt_info, slp_node, 0, - vect_body); + slp_node, 0, vect_body); else { tree vec_inv = build_constructor (lvectype, v); @@ -11012,36 +10353,27 @@ vectorizable_load (vec_info *vinfo, if (!costing_p) { - if (slp) - { - if (slp_perm) - dr_chain.quick_push (gimple_assign_lhs (new_stmt)); - else - slp_node->push_vec_def (new_stmt); - } + if (ls.slp_perm) + dr_chain.quick_push (gimple_assign_lhs (new_stmt)); else - { - if (j == 0) - *vec_stmt = new_stmt; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); - } + slp_node->push_vec_def (new_stmt); } } - if (slp_perm) + if (ls.slp_perm) { - unsigned n_perms; if (costing_p) { - unsigned n_loads; - vect_transform_slp_perm_load (vinfo, slp_node, vNULL, NULL, vf, - true, &n_perms, &n_loads); - inside_cost += record_stmt_cost (cost_vec, n_perms, vec_perm, - first_stmt_info, slp_node, 0, - vect_body); + gcc_assert (ls.n_perms != -1U); + inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm, + slp_node, 0, vect_body); } else - vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf, - false, &n_perms); + { + unsigned n_perms2; + vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf, + false, &n_perms2); + gcc_assert (ls.n_perms == n_perms2); + } } if (costing_p) @@ -11061,12 +10393,12 @@ vectorizable_load (vec_info *vinfo, return true; } - if (memory_access_type == VMAT_GATHER_SCATTER - || (!slp && memory_access_type == VMAT_CONTIGUOUS)) + if (mat_gather_scatter_p (memory_access_type) + && !ls.ls_type) grouped_load = false; if (grouped_load - || (slp && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())) + || SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) { if (grouped_load) { @@ -11080,7 +10412,7 @@ vectorizable_load (vec_info *vinfo, } /* For SLP vectorization we directly vectorize a subchain without permutation. */ - if (slp && ! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) + if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0]; /* For BB vectorization always use the first stmt to base the data ref pointer on. */ @@ -11088,60 +10420,38 @@ vectorizable_load (vec_info *vinfo, first_stmt_info_for_drptr = vect_find_first_scalar_stmt_in_slp (slp_node); - /* Check if the chain of loads is already vectorized. */ - if (STMT_VINFO_VEC_STMTS (first_stmt_info).exists () - /* For SLP we would need to copy over SLP_TREE_VEC_DEFS. - ??? But we can only do so if there is exactly one - as we have no way to get at the rest. Leave the CSE - opportunity alone. - ??? With the group load eventually participating - in multiple different permutations (having multiple - slp nodes which refer to the same group) the CSE - is even wrong code. See PR56270. */ - && !slp) - { - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - return true; - } first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info); group_gap_adj = 0; /* VEC_NUM is the number of vect stmts to be created for this group. */ - if (slp) - { - grouped_load = false; - /* If an SLP permutation is from N elements to N elements, - and if one vector holds a whole number of N, we can load - the inputs to the permutation in the same way as an - unpermuted sequence. In other cases we need to load the - whole group, not only the number of vector stmts the - permutation result fits in. */ - unsigned scalar_lanes = SLP_TREE_LANES (slp_node); - if (nested_in_vect_loop) - /* We do not support grouped accesses in a nested loop, - instead the access is contiguous but it might be - permuted. No gap adjustment is needed though. */ - vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); - else if (slp_perm - && (group_size != scalar_lanes - || !multiple_p (nunits, group_size))) - { - /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for - variable VF; see vect_transform_slp_perm_load. */ - unsigned int const_vf = vf.to_constant (); - unsigned int const_nunits = nunits.to_constant (); - vec_num = CEIL (group_size * const_vf, const_nunits); - group_gap_adj = vf * group_size - nunits * vec_num; - } - else - { - vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); - group_gap_adj - = group_size - scalar_lanes; - } - } + grouped_load = false; + /* If an SLP permutation is from N elements to N elements, + and if one vector holds a whole number of N, we can load + the inputs to the permutation in the same way as an + unpermuted sequence. In other cases we need to load the + whole group, not only the number of vector stmts the + permutation result fits in. */ + unsigned scalar_lanes = SLP_TREE_LANES (slp_node); + if (nested_in_vect_loop) + /* We do not support grouped accesses in a nested loop, + instead the access is contiguous but it might be + permuted. No gap adjustment is needed though. */ + ; + else if (ls.slp_perm + && (group_size != scalar_lanes + || !multiple_p (nunits, group_size))) + { + /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for + variable VF; see vect_transform_slp_perm_load. */ + unsigned int const_vf = vf.to_constant (); + unsigned int const_nunits = nunits.to_constant (); + vec_num = CEIL (group_size * const_vf, const_nunits); + group_gap_adj = vf * group_size - nunits * vec_num; + } else - vec_num = group_size; + { + group_gap_adj = group_size - scalar_lanes; + } ref_type = get_group_alias_ptr_type (first_stmt_info); } @@ -11149,14 +10459,11 @@ vectorizable_load (vec_info *vinfo, { first_stmt_info = stmt_info; first_dr_info = dr_info; - group_size = vec_num = 1; + group_size = 1; group_gap_adj = 0; ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr)); - if (slp) - vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); } - gcc_assert (alignment_support_scheme); vec_loop_masks *loop_masks = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo) ? &LOOP_VINFO_MASKS (loop_vinfo) @@ -11176,10 +10483,12 @@ vectorizable_load (vec_info *vinfo, /* Targets with store-lane instructions must not require explicit realignment. vect_supportable_dr_alignment always returns either - dr_aligned or dr_unaligned_supported for masked operations. */ + dr_aligned or dr_unaligned_supported for (non-length) masked + operations. */ gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES - && !mask + && !mask_node && !loop_masks) + || mat_gather_scatter_p (memory_access_type) || alignment_support_scheme == dr_aligned || alignment_support_scheme == dr_unaligned_supported); @@ -11215,39 +10524,6 @@ vectorizable_load (vec_info *vinfo, S2: z = x + 1 - - */ - /* In case of interleaving (non-unit grouped access): - - S1: x2 = &base + 2 - S2: x0 = &base - S3: x1 = &base + 1 - S4: x3 = &base + 3 - - Vectorized loads are created in the order of memory accesses - starting from the access of the first stmt of the chain: - - VS1: vx0 = &base - VS2: vx1 = &base + vec_size*1 - VS3: vx3 = &base + vec_size*2 - VS4: vx4 = &base + vec_size*3 - - Then permutation statements are generated: - - VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } > - VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } > - ... - - And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts - (the order of the data-refs in the output of vect_permute_load_chain - corresponds to the order of scalar stmts in the interleaving chain - see - the documentation of vect_permute_load_chain()). - The generation of permutation stmts and recording them in - STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load(). - - In case of both multiple types and interleaving, the vector loads and - permutation stmts above are created for every copy. The result vector - stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the - corresponding STMT_VINFO_RELATED_STMT for the next copies. */ - /* If the data reference is aligned (dr_aligned) or potentially unaligned on a target that supports unaligned accesses (dr_unaligned_supported) we generate the following code: @@ -11306,7 +10582,7 @@ vectorizable_load (vec_info *vinfo, if (!costing_p) { if (!diff_first_stmt_info) - msq = vect_setup_realignment (vinfo, first_stmt_info, gsi, + msq = vect_setup_realignment (vinfo, first_stmt_info, vectype, gsi, &realignment_token, alignment_support_scheme, NULL_TREE, &at_loop); @@ -11329,54 +10605,32 @@ vectorizable_load (vec_info *vinfo, tree bump; tree vec_offset = NULL_TREE; - if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) - { - aggr_type = NULL_TREE; - bump = NULL_TREE; - } - else if (memory_access_type == VMAT_GATHER_SCATTER) - { - aggr_type = elem_type; - if (!costing_p) - vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info, - &bump, &vec_offset, loop_lens); - } - else - { - if (memory_access_type == VMAT_LOAD_STORE_LANES) - aggr_type = build_array_type_nelts (elem_type, group_size * nunits); - else - aggr_type = vectype; - if (!costing_p) - bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type, - memory_access_type, loop_lens); - } auto_vec<tree> vec_offsets; auto_vec<tree> vec_masks; - if (mask && !costing_p) - { - if (slp_node) - vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[mask_index], - &vec_masks); - else - vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies, mask, - &vec_masks, mask_vectype); - } + if (mask_node && !costing_p) + vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[mask_index], + &vec_masks); tree vec_mask = NULL_TREE; tree vec_els = NULL_TREE; if (memory_access_type == VMAT_LOAD_STORE_LANES) { + const internal_fn lanes_ifn = ls.lanes_ifn; + gcc_assert (alignment_support_scheme == dr_aligned || alignment_support_scheme == dr_unaligned_supported); + aggr_type = build_array_type_nelts (elem_type, group_size * nunits); + if (!costing_p) + bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type, + memory_access_type, loop_lens); + unsigned int inside_cost = 0, prologue_cost = 0; /* For costing some adjacent vector loads, we'd like to cost with the total number of them once instead of cost each one by one. */ unsigned int n_adjacent_loads = 0; - if (slp_node) - ncopies = slp_node->vec_stmts_size / group_size; + int ncopies = vec_num / group_size; for (j = 0; j < ncopies; j++) { if (costing_p) @@ -11424,7 +10678,7 @@ vectorizable_load (vec_info *vinfo, dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, stmt_info, bump); } - if (mask) + if (mask_node) vec_mask = vec_masks[j]; tree vec_array = create_vector_array (vectype, group_size); @@ -11499,32 +10753,17 @@ vectorizable_load (vec_info *vinfo, gimple_call_set_nothrow (call, true); vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); - if (!slp) - dr_chain.create (group_size); /* Extract each vector into an SSA_NAME. */ for (unsigned i = 0; i < group_size; i++) { new_temp = read_vector_array (vinfo, stmt_info, gsi, scalar_dest, vec_array, i, need_zeroing, final_mask); - if (slp) - slp_node->push_vec_def (new_temp); - else - dr_chain.quick_push (new_temp); + slp_node->push_vec_def (new_temp); } - if (!slp) - /* Record the mapping between SSA_NAMEs and statements. */ - vect_record_grouped_load_vectors (vinfo, stmt_info, dr_chain); - /* Record that VEC_ARRAY is now dead. */ vect_clobber_variable (vinfo, stmt_info, gsi, vec_array); - - if (!slp) - dr_chain.release (); - - if (!slp_node) - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; } if (costing_p) @@ -11544,371 +10783,428 @@ vectorizable_load (vec_info *vinfo, return true; } - if (memory_access_type == VMAT_GATHER_SCATTER) + if (mat_gather_scatter_p (memory_access_type)) { - gcc_assert (alignment_support_scheme == dr_aligned - || alignment_support_scheme == dr_unaligned_supported); - gcc_assert (!grouped_load && !slp_perm); + gcc_assert ((!grouped_load && !ls.slp_perm) || ls.ls_type); - unsigned int inside_cost = 0, prologue_cost = 0; - for (j = 0; j < ncopies; j++) + auto_vec<tree> dr_chain (vec_num); + + /* If we pun the original vectype the loads as well as costing, length, + etc. is performed with the new type. After loading we VIEW_CONVERT + the data to the original vectype. */ + tree original_vectype = vectype; + if (ls.ls_type) + vectype = ls.ls_type; + + /* 1. Create the vector or array pointer update chain. */ + if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { - /* 1. Create the vector or array pointer update chain. */ - if (j == 0 && !costing_p) + aggr_type = NULL_TREE; + bump = NULL_TREE; + if (!costing_p) + vect_get_gather_scatter_ops (loop, slp_node, &dataref_ptr, + &vec_offsets); + } + else + { + aggr_type = elem_type; + if (!costing_p) { - if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) - vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info, - slp_node, &gs_info, &dataref_ptr, - &vec_offsets); - else - dataref_ptr + vect_get_strided_load_store_ops (stmt_info, slp_node, vectype, + ls.strided_offset_vectype, + loop_vinfo, gsi, + &bump, &vec_offset, loop_lens); + dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type, at_loop, offset, &dummy, gsi, &ptr_incr, false, bump); } - else if (!costing_p) + } + + unsigned int inside_cost = 0, prologue_cost = 0; + + gimple *new_stmt = NULL; + for (i = 0; i < vec_num; i++) + { + tree final_mask = NULL_TREE; + tree final_len = NULL_TREE; + tree bias = NULL_TREE; + if (!costing_p) { - gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)); - if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info)) + if (mask_node) + vec_mask = vec_masks[i]; + if (loop_masks) + final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks, + vec_num, vectype, i); + if (vec_mask) + final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, + final_mask, vec_mask, gsi); + + if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)) dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, stmt_info, bump); } - gimple *new_stmt = NULL; - for (i = 0; i < vec_num; i++) + /* 2. Create the vector-load in the loop. */ + unsigned align = get_object_alignment (DR_REF (first_dr_info->dr)); + tree alias_align_ptr = build_int_cst (ref_type, align); + if (memory_access_type == VMAT_GATHER_SCATTER_IFN) { - tree final_mask = NULL_TREE; - tree final_len = NULL_TREE; - tree bias = NULL_TREE; - if (!costing_p) + if (costing_p) { - if (mask) - vec_mask = vec_masks[vec_num * j + i]; - if (loop_masks) - final_mask - = vect_get_loop_mask (loop_vinfo, gsi, loop_masks, - vec_num * ncopies, vectype, - vec_num * j + i); - if (vec_mask) - final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, - final_mask, vec_mask, gsi); - - if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)) - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, - gsi, stmt_info, bump); - } + if (ls.supported_offset_vectype) + inside_cost + += record_stmt_cost (cost_vec, 1, vector_stmt, + slp_node, 0, vect_body); + if (ls.supported_scale) + inside_cost + += record_stmt_cost (cost_vec, 1, vector_stmt, + slp_node, 0, vect_body); - /* 2. Create the vector-load in the loop. */ - unsigned HOST_WIDE_INT align; - if (gs_info.ifn != IFN_LAST) + unsigned int cnunits = vect_nunits_for_cost (vectype); + inside_cost + = record_stmt_cost (cost_vec, cnunits, scalar_load, + slp_node, 0, vect_body); + continue; + } + if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) + vec_offset = vec_offsets[i]; + tree zero = build_zero_cst (vectype); + tree scale = size_int (SLP_TREE_GS_SCALE (slp_node)); + bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset)); + + /* Perform the offset conversion and scaling if necessary. */ + if (!strided + && (ls.supported_offset_vectype || ls.supported_scale)) { - if (costing_p) + gimple_seq stmts = NULL; + if (ls.supported_offset_vectype) + vec_offset = gimple_convert + (&stmts, ls.supported_offset_vectype, vec_offset); + if (ls.supported_scale) { - unsigned int cnunits = vect_nunits_for_cost (vectype); - inside_cost - = record_stmt_cost (cost_vec, cnunits, scalar_load, - stmt_info, slp_node, 0, vect_body); - continue; + tree mult_cst = build_int_cst + (TREE_TYPE (TREE_TYPE (vec_offset)), + SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale); + tree mult = build_vector_from_val + (TREE_TYPE (vec_offset), mult_cst); + vec_offset = gimple_build + (&stmts, MULT_EXPR, TREE_TYPE (vec_offset), + vec_offset, mult); + scale = size_int (ls.supported_scale); } - if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) - vec_offset = vec_offsets[vec_num * j + i]; - tree zero = build_zero_cst (vectype); - tree scale = size_int (gs_info.scale); + gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT); + } - if (gs_info.ifn == IFN_MASK_LEN_GATHER_LOAD) + if (ls.gs.ifn == IFN_MASK_LEN_GATHER_LOAD) + { + if (loop_lens) + final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens, + vec_num, vectype, i, 1); + else + final_len = build_int_cst (sizetype, + TYPE_VECTOR_SUBPARTS (vectype)); + signed char biasval + = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); + bias = build_int_cst (intQI_type_node, biasval); + if (!final_mask) { - if (loop_lens) - final_len - = vect_get_loop_len (loop_vinfo, gsi, loop_lens, - vec_num * ncopies, vectype, - vec_num * j + i, 1); - else - final_len - = build_int_cst (sizetype, - TYPE_VECTOR_SUBPARTS (vectype)); - signed char biasval - = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); - bias = build_int_cst (intQI_type_node, biasval); - if (!final_mask) - { - mask_vectype = truth_type_for (vectype); - final_mask = build_minus_one_cst (mask_vectype); - } + mask_vectype = truth_type_for (vectype); + final_mask = build_minus_one_cst (mask_vectype); } + } - if (final_mask) - { - vec_els = vect_get_mask_load_else - (maskload_elsval, vectype); - if (type_mode_padding_p - && maskload_elsval != MASK_LOAD_ELSE_ZERO) - need_zeroing = true; - } + if (final_mask) + { + vec_els = vect_get_mask_load_else (maskload_elsval, vectype); + if (type_mode_padding_p + && maskload_elsval != MASK_LOAD_ELSE_ZERO) + need_zeroing = true; + } - gcall *call; - if (final_len && final_mask) - { - if (VECTOR_TYPE_P (TREE_TYPE (vec_offset))) - call = gimple_build_call_internal ( - IFN_MASK_LEN_GATHER_LOAD, 8, dataref_ptr, vec_offset, - scale, zero, final_mask, vec_els, final_len, bias); - else - /* Non-vector offset indicates that prefer to take - MASK_LEN_STRIDED_LOAD instead of the - MASK_LEN_GATHER_LOAD with direct stride arg. */ - call = gimple_build_call_internal ( - IFN_MASK_LEN_STRIDED_LOAD, 7, dataref_ptr, vec_offset, - zero, final_mask, vec_els, final_len, bias); - } - else if (final_mask) - call = gimple_build_call_internal (IFN_MASK_GATHER_LOAD, - 6, dataref_ptr, - vec_offset, scale, - zero, final_mask, - vec_els); + gcall *call; + if (final_len && final_mask) + { + if (VECTOR_TYPE_P (TREE_TYPE (vec_offset))) + call = gimple_build_call_internal (IFN_MASK_LEN_GATHER_LOAD, + 9, dataref_ptr, + alias_align_ptr, + vec_offset, scale, zero, + final_mask, vec_els, + final_len, bias); else - call = gimple_build_call_internal (IFN_GATHER_LOAD, 4, - dataref_ptr, vec_offset, - scale, zero); - gimple_call_set_nothrow (call, true); - new_stmt = call; + /* Non-vector offset indicates that prefer to take + MASK_LEN_STRIDED_LOAD instead of the + MASK_LEN_GATHER_LOAD with direct stride arg. */ + call = gimple_build_call_internal + (IFN_MASK_LEN_STRIDED_LOAD, 7, dataref_ptr, + vec_offset, zero, final_mask, vec_els, final_len, + bias); + } + else if (final_mask) + call = gimple_build_call_internal (IFN_MASK_GATHER_LOAD, + 7, dataref_ptr, + alias_align_ptr, + vec_offset, scale, + zero, final_mask, vec_els); + else + call = gimple_build_call_internal (IFN_GATHER_LOAD, 5, + dataref_ptr, + alias_align_ptr, + vec_offset, scale, zero); + gimple_call_set_nothrow (call, true); + new_stmt = call; + data_ref = NULL_TREE; + } + else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY) + { + /* The builtin decls path for gather is legacy, x86 only. */ + gcc_assert (!final_len && nunits.is_constant ()); + if (costing_p) + { + unsigned int cnunits = vect_nunits_for_cost (vectype); + inside_cost + = record_stmt_cost (cost_vec, cnunits, scalar_load, + slp_node, 0, vect_body); + continue; + } + tree offset_vectype = TREE_TYPE (vec_offsets[0]); + poly_uint64 offset_nunits = TYPE_VECTOR_SUBPARTS (offset_vectype); + if (known_eq (nunits, offset_nunits)) + { + new_stmt = vect_build_one_gather_load_call + (vinfo, stmt_info, slp_node, vectype, gsi, + ls.gs.decl, dataref_ptr, vec_offsets[i], + final_mask); data_ref = NULL_TREE; } - else if (gs_info.decl) + else if (known_eq (nunits, offset_nunits * 2)) { - /* The builtin decls path for gather is legacy, x86 only. */ - gcc_assert (!final_len && nunits.is_constant ()); - if (costing_p) - { - unsigned int cnunits = vect_nunits_for_cost (vectype); - inside_cost - = record_stmt_cost (cost_vec, cnunits, scalar_load, - stmt_info, slp_node, 0, vect_body); - continue; - } - poly_uint64 offset_nunits - = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype); - if (known_eq (nunits, offset_nunits)) + /* We have a offset vector with half the number of + lanes but the builtins will produce full vectype + data with just the lower lanes filled. */ + new_stmt = vect_build_one_gather_load_call + (vinfo, stmt_info, slp_node, vectype, gsi, + ls.gs.decl, dataref_ptr, vec_offsets[2 * i], + final_mask); + tree low = make_ssa_name (vectype); + gimple_set_lhs (new_stmt, low); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + + /* now put upper half of final_mask in final_mask low. */ + if (final_mask + && !SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (final_mask)))) { - new_stmt = vect_build_one_gather_load_call - (vinfo, stmt_info, gsi, &gs_info, - dataref_ptr, vec_offsets[vec_num * j + i], - final_mask); - data_ref = NULL_TREE; + int count = nunits.to_constant (); + vec_perm_builder sel (count, count, 1); + sel.quick_grow (count); + for (int i = 0; i < count; ++i) + sel[i] = i | (count / 2); + vec_perm_indices indices (sel, 2, count); + tree perm_mask = vect_gen_perm_mask_checked + (TREE_TYPE (final_mask), indices); + new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR, + final_mask, final_mask, + perm_mask); + final_mask = make_ssa_name (TREE_TYPE (final_mask)); + gimple_set_lhs (new_stmt, final_mask); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); } - else if (known_eq (nunits, offset_nunits * 2)) + else if (final_mask) { - /* We have a offset vector with half the number of - lanes but the builtins will produce full vectype - data with just the lower lanes filled. */ - new_stmt = vect_build_one_gather_load_call - (vinfo, stmt_info, gsi, &gs_info, - dataref_ptr, vec_offsets[2 * vec_num * j + 2 * i], - final_mask); - tree low = make_ssa_name (vectype); - gimple_set_lhs (new_stmt, low); + new_stmt = gimple_build_assign (NULL_TREE, + VEC_UNPACK_HI_EXPR, + final_mask); + final_mask = make_ssa_name + (truth_type_for (offset_vectype)); + gimple_set_lhs (new_stmt, final_mask); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + } - /* now put upper half of final_mask in final_mask low. */ - if (final_mask - && !SCALAR_INT_MODE_P - (TYPE_MODE (TREE_TYPE (final_mask)))) - { - int count = nunits.to_constant (); - vec_perm_builder sel (count, count, 1); - sel.quick_grow (count); - for (int i = 0; i < count; ++i) - sel[i] = i | (count / 2); - vec_perm_indices indices (sel, 2, count); - tree perm_mask = vect_gen_perm_mask_checked - (TREE_TYPE (final_mask), indices); - new_stmt = gimple_build_assign (NULL_TREE, - VEC_PERM_EXPR, - final_mask, - final_mask, - perm_mask); - final_mask = make_ssa_name (TREE_TYPE (final_mask)); - gimple_set_lhs (new_stmt, final_mask); - vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); - } - else if (final_mask) - { - new_stmt = gimple_build_assign (NULL_TREE, - VEC_UNPACK_HI_EXPR, - final_mask); - final_mask = make_ssa_name - (truth_type_for (gs_info.offset_vectype)); - gimple_set_lhs (new_stmt, final_mask); - vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); - } - - new_stmt = vect_build_one_gather_load_call - (vinfo, stmt_info, gsi, &gs_info, - dataref_ptr, - vec_offsets[2 * vec_num * j + 2 * i + 1], - final_mask); - tree high = make_ssa_name (vectype); - gimple_set_lhs (new_stmt, high); - vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + new_stmt = vect_build_one_gather_load_call + (vinfo, stmt_info, slp_node, vectype, gsi, + ls.gs.decl, dataref_ptr, + vec_offsets[2 * i + 1], final_mask); + tree high = make_ssa_name (vectype); + gimple_set_lhs (new_stmt, high); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - /* compose low + high. */ - int count = nunits.to_constant (); + /* compose low + high. */ + int count = nunits.to_constant (); + vec_perm_builder sel (count, count, 1); + sel.quick_grow (count); + for (int i = 0; i < count; ++i) + sel[i] = i < count / 2 ? i : i + count / 2; + vec_perm_indices indices (sel, 2, count); + tree perm_mask + = vect_gen_perm_mask_checked (vectype, indices); + new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR, + low, high, perm_mask); + data_ref = NULL_TREE; + } + else if (known_eq (nunits * 2, offset_nunits)) + { + /* We have a offset vector with double the number of + lanes. Select the low/high part accordingly. */ + vec_offset = vec_offsets[i / 2]; + if (i & 1) + { + int count = offset_nunits.to_constant (); vec_perm_builder sel (count, count, 1); sel.quick_grow (count); for (int i = 0; i < count; ++i) - sel[i] = i < count / 2 ? i : i + count / 2; + sel[i] = i | (count / 2); vec_perm_indices indices (sel, 2, count); - tree perm_mask - = vect_gen_perm_mask_checked (vectype, indices); - new_stmt = gimple_build_assign (NULL_TREE, - VEC_PERM_EXPR, - low, high, perm_mask); - data_ref = NULL_TREE; - } - else if (known_eq (nunits * 2, offset_nunits)) - { - /* We have a offset vector with double the number of - lanes. Select the low/high part accordingly. */ - vec_offset = vec_offsets[(vec_num * j + i) / 2]; - if ((vec_num * j + i) & 1) - { - int count = offset_nunits.to_constant (); - vec_perm_builder sel (count, count, 1); - sel.quick_grow (count); - for (int i = 0; i < count; ++i) - sel[i] = i | (count / 2); - vec_perm_indices indices (sel, 2, count); - tree perm_mask = vect_gen_perm_mask_checked - (TREE_TYPE (vec_offset), indices); - new_stmt = gimple_build_assign (NULL_TREE, - VEC_PERM_EXPR, - vec_offset, - vec_offset, - perm_mask); - vec_offset = make_ssa_name (TREE_TYPE (vec_offset)); - gimple_set_lhs (new_stmt, vec_offset); - vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); - } - new_stmt = vect_build_one_gather_load_call - (vinfo, stmt_info, gsi, &gs_info, - dataref_ptr, vec_offset, final_mask); - data_ref = NULL_TREE; + tree perm_mask = vect_gen_perm_mask_checked + (TREE_TYPE (vec_offset), indices); + new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR, + vec_offset, vec_offset, + perm_mask); + vec_offset = make_ssa_name (TREE_TYPE (vec_offset)); + gimple_set_lhs (new_stmt, vec_offset); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); } - else - gcc_unreachable (); + new_stmt = vect_build_one_gather_load_call + (vinfo, stmt_info, slp_node, vectype, gsi, + ls.gs.decl, + dataref_ptr, vec_offset, final_mask); + data_ref = NULL_TREE; } else + gcc_unreachable (); + } + else + { + /* Emulated gather-scatter. */ + gcc_assert (!final_mask); + unsigned HOST_WIDE_INT const_nunits = nunits.to_constant (); + if (costing_p) { - /* Emulated gather-scatter. */ - gcc_assert (!final_mask); - unsigned HOST_WIDE_INT const_nunits = nunits.to_constant (); - if (costing_p) - { - /* For emulated gathers N offset vector element - offset add is consumed by the load). */ - inside_cost = record_stmt_cost (cost_vec, const_nunits, - vec_to_scalar, stmt_info, - slp_node, 0, vect_body); - /* N scalar loads plus gathering them into a - vector. */ - inside_cost - = record_stmt_cost (cost_vec, const_nunits, scalar_load, - stmt_info, slp_node, 0, vect_body); - inside_cost - = record_stmt_cost (cost_vec, 1, vec_construct, - stmt_info, slp_node, 0, vect_body); - continue; - } - unsigned HOST_WIDE_INT const_offset_nunits - = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype) - .to_constant (); - vec<constructor_elt, va_gc> *ctor_elts; - vec_alloc (ctor_elts, const_nunits); - gimple_seq stmts = NULL; - /* We support offset vectors with more elements - than the data vector for now. */ - unsigned HOST_WIDE_INT factor - = const_offset_nunits / const_nunits; - vec_offset = vec_offsets[(vec_num * j + i) / factor]; - unsigned elt_offset - = ((vec_num * j + i) % factor) * const_nunits; - tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset)); - tree scale = size_int (gs_info.scale); - align = get_object_alignment (DR_REF (first_dr_info->dr)); - tree ltype = build_aligned_type (TREE_TYPE (vectype), align); - for (unsigned k = 0; k < const_nunits; ++k) - { - tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type), - bitsize_int (k + elt_offset)); - tree idx - = gimple_build (&stmts, BIT_FIELD_REF, idx_type, - vec_offset, TYPE_SIZE (idx_type), boff); - idx = gimple_convert (&stmts, sizetype, idx); - idx = gimple_build (&stmts, MULT_EXPR, sizetype, idx, - scale); - tree ptr = gimple_build (&stmts, PLUS_EXPR, - TREE_TYPE (dataref_ptr), - dataref_ptr, idx); - ptr = gimple_convert (&stmts, ptr_type_node, ptr); - tree elt = make_ssa_name (TREE_TYPE (vectype)); - tree ref = build2 (MEM_REF, ltype, ptr, - build_int_cst (ref_type, 0)); - new_stmt = gimple_build_assign (elt, ref); - gimple_set_vuse (new_stmt, gimple_vuse (gsi_stmt (*gsi))); - gimple_seq_add_stmt (&stmts, new_stmt); - CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE, elt); - } - gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT); - new_stmt = gimple_build_assign ( - NULL_TREE, build_constructor (vectype, ctor_elts)); - data_ref = NULL_TREE; + /* For emulated gathers N offset vector element + offset add is consumed by the load). */ + inside_cost = record_stmt_cost (cost_vec, const_nunits, + vec_to_scalar, + slp_node, 0, vect_body); + /* N scalar loads plus gathering them into a + vector. */ + inside_cost + = record_stmt_cost (cost_vec, const_nunits, scalar_load, + slp_node, 0, vect_body); + inside_cost + = record_stmt_cost (cost_vec, 1, vec_construct, + slp_node, 0, vect_body); + continue; } - - vec_dest = vect_create_destination_var (scalar_dest, vectype); - /* DATA_REF is null if we've already built the statement. */ - if (data_ref) + tree offset_vectype = TREE_TYPE (vec_offsets[0]); + unsigned HOST_WIDE_INT const_offset_nunits + = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant (); + vec<constructor_elt, va_gc> *ctor_elts; + vec_alloc (ctor_elts, const_nunits); + gimple_seq stmts = NULL; + /* We support offset vectors with more elements + than the data vector for now. */ + unsigned HOST_WIDE_INT factor + = const_offset_nunits / const_nunits; + vec_offset = vec_offsets[i / factor]; + unsigned elt_offset = (i % factor) * const_nunits; + tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset)); + tree scale = size_int (SLP_TREE_GS_SCALE (slp_node)); + tree ltype = build_aligned_type (TREE_TYPE (vectype), align); + for (unsigned k = 0; k < const_nunits; ++k) { - vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); - new_stmt = gimple_build_assign (vec_dest, data_ref); + tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type), + bitsize_int (k + elt_offset)); + tree idx = gimple_build (&stmts, BIT_FIELD_REF, idx_type, + vec_offset, TYPE_SIZE (idx_type), + boff); + idx = gimple_convert (&stmts, sizetype, idx); + idx = gimple_build (&stmts, MULT_EXPR, sizetype, idx, scale); + tree ptr = gimple_build (&stmts, PLUS_EXPR, + TREE_TYPE (dataref_ptr), + dataref_ptr, idx); + ptr = gimple_convert (&stmts, ptr_type_node, ptr); + tree elt = make_ssa_name (TREE_TYPE (vectype)); + tree ref = build2 (MEM_REF, ltype, ptr, + build_int_cst (ref_type, 0)); + new_stmt = gimple_build_assign (elt, ref); + gimple_set_vuse (new_stmt, gimple_vuse (gsi_stmt (*gsi))); + gimple_seq_add_stmt (&stmts, new_stmt); + CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE, elt); } - new_temp = need_zeroing - ? make_ssa_name (vectype) - : make_ssa_name (vec_dest, new_stmt); - gimple_set_lhs (new_stmt, new_temp); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT); + new_stmt = gimple_build_assign (NULL_TREE, + build_constructor (vectype, + ctor_elts)); + data_ref = NULL_TREE; + } - /* If we need to explicitly zero inactive elements emit a - VEC_COND_EXPR that does so. */ - if (need_zeroing) - { - vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO, - vectype); + vec_dest = vect_create_destination_var (scalar_dest, vectype); + /* DATA_REF is null if we've already built the statement. */ + if (data_ref) + { + vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); + new_stmt = gimple_build_assign (vec_dest, data_ref); + } + new_temp = (need_zeroing + ? make_ssa_name (vectype) + : make_ssa_name (vec_dest, new_stmt)); + gimple_set_lhs (new_stmt, new_temp); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - tree new_temp2 = make_ssa_name (vec_dest, new_stmt); - new_stmt - = gimple_build_assign (new_temp2, VEC_COND_EXPR, - final_mask, new_temp, vec_els); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, - gsi); - new_temp = new_temp2; - } + /* If we need to explicitly zero inactive elements emit a + VEC_COND_EXPR that does so. */ + if (need_zeroing) + { + vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO, + vectype); - /* Store vector loads in the corresponding SLP_NODE. */ - if (slp) - slp_node->push_vec_def (new_stmt); + tree new_temp2 = make_ssa_name (vec_dest, new_stmt); + new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR, + final_mask, new_temp, vec_els); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_temp = new_temp2; } - if (!slp && !costing_p) - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + if (ls.ls_type) + { + new_stmt = gimple_build_assign (make_ssa_name + (original_vectype), + VIEW_CONVERT_EXPR, + build1 (VIEW_CONVERT_EXPR, + original_vectype, + new_temp)); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + } + + /* Store vector loads in the corresponding SLP_NODE. */ + if (!costing_p) + { + if (ls.slp_perm) + dr_chain.quick_push (gimple_assign_lhs (new_stmt)); + else + slp_node->push_vec_def (new_stmt); + } } - if (!slp && !costing_p) - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; + if (ls.slp_perm) + { + if (costing_p) + { + gcc_assert (ls.n_perms != -1U); + inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm, + slp_node, 0, vect_body); + } + else + { + unsigned n_perms2; + vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf, + false, &n_perms2); + gcc_assert (ls.n_perms == n_perms2); + } + } if (costing_p && dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -11918,278 +11214,313 @@ vectorizable_load (vec_info *vinfo, return true; } + aggr_type = vectype; + if (!costing_p) + bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type, + memory_access_type, loop_lens); + poly_uint64 group_elt = 0; unsigned int inside_cost = 0, prologue_cost = 0; /* For costing some adjacent vector loads, we'd like to cost with the total number of them once instead of cost each one by one. */ unsigned int n_adjacent_loads = 0; - for (j = 0; j < ncopies; j++) - { - /* 1. Create the vector or array pointer update chain. */ - if (j == 0 && !costing_p) - { - bool simd_lane_access_p - = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0; - if (simd_lane_access_p - && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR - && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0)) - && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info)) - && integer_zerop (DR_INIT (first_dr_info->dr)) - && alias_sets_conflict_p (get_alias_set (aggr_type), - get_alias_set (TREE_TYPE (ref_type))) - && (alignment_support_scheme == dr_aligned - || alignment_support_scheme == dr_unaligned_supported)) - { - dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr)); - dataref_offset = build_int_cst (ref_type, 0); - } - else if (diff_first_stmt_info) + + /* 1. Create the vector or array pointer update chain. */ + if (!costing_p) + { + bool simd_lane_access_p + = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0; + if (simd_lane_access_p + && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR + && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0)) + && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info)) + && integer_zerop (DR_INIT (first_dr_info->dr)) + && alias_sets_conflict_p (get_alias_set (aggr_type), + get_alias_set (TREE_TYPE (ref_type))) + && (alignment_support_scheme == dr_aligned + || alignment_support_scheme == dr_unaligned_supported)) + { + dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr)); + dataref_offset = build_int_cst (ref_type, 0); + } + else if (diff_first_stmt_info) + { + dataref_ptr + = vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr, + aggr_type, at_loop, offset, &dummy, + gsi, &ptr_incr, simd_lane_access_p, + bump); + /* Adjust the pointer by the difference to first_stmt. */ + data_reference_p ptrdr + = STMT_VINFO_DATA_REF (first_stmt_info_for_drptr); + tree diff = fold_convert (sizetype, + size_binop (MINUS_EXPR, + DR_INIT (first_dr_info->dr), + DR_INIT (ptrdr))); + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, + stmt_info, diff); + if (alignment_support_scheme == dr_explicit_realign) { - dataref_ptr - = vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr, - aggr_type, at_loop, offset, &dummy, - gsi, &ptr_incr, simd_lane_access_p, - bump); - /* Adjust the pointer by the difference to first_stmt. */ - data_reference_p ptrdr - = STMT_VINFO_DATA_REF (first_stmt_info_for_drptr); - tree diff - = fold_convert (sizetype, - size_binop (MINUS_EXPR, - DR_INIT (first_dr_info->dr), - DR_INIT (ptrdr))); - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, - stmt_info, diff); - if (alignment_support_scheme == dr_explicit_realign) - { - msq = vect_setup_realignment (vinfo, - first_stmt_info_for_drptr, gsi, - &realignment_token, - alignment_support_scheme, - dataref_ptr, &at_loop); - gcc_assert (!compute_in_loop); - } + msq = vect_setup_realignment (vinfo, first_stmt_info_for_drptr, + vectype, gsi, + &realignment_token, + alignment_support_scheme, + dataref_ptr, &at_loop); + gcc_assert (!compute_in_loop); } - else - dataref_ptr - = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type, - at_loop, - offset, &dummy, gsi, &ptr_incr, - simd_lane_access_p, bump); - } - else if (!costing_p) - { - gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)); - if (dataref_offset) - dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, - bump); - else - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, - stmt_info, bump); } + else + dataref_ptr + = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type, + at_loop, + offset, &dummy, gsi, &ptr_incr, + simd_lane_access_p, bump); + } + else if (!costing_p) + { + gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)); + if (dataref_offset) + dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump); + else + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, + stmt_info, bump); + } - if (grouped_load || slp_perm) - dr_chain.create (vec_num); + auto_vec<tree> dr_chain; + if (grouped_load || ls.slp_perm) + dr_chain.create (vec_num); - gimple *new_stmt = NULL; - for (i = 0; i < vec_num; i++) + gimple *new_stmt = NULL; + for (i = 0; i < vec_num; i++) + { + tree final_mask = NULL_TREE; + tree final_len = NULL_TREE; + tree bias = NULL_TREE; + + if (!costing_p) { - tree final_mask = NULL_TREE; - tree final_len = NULL_TREE; - tree bias = NULL_TREE; + if (mask_node) + vec_mask = vec_masks[i]; + if (loop_masks) + final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks, + vec_num, vectype, i); + if (vec_mask) + final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, + final_mask, vec_mask, gsi); - if (!costing_p) - { - if (mask) - vec_mask = vec_masks[vec_num * j + i]; - if (loop_masks) - final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks, - vec_num * ncopies, vectype, - vec_num * j + i); - if (vec_mask) - final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, - final_mask, vec_mask, gsi); + if (i > 0) + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, + gsi, stmt_info, bump); + } - if (i > 0) - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, - gsi, stmt_info, bump); - } + /* 2. Create the vector-load in the loop. */ + switch (alignment_support_scheme) + { + case dr_aligned: + case dr_unaligned_supported: + { + if (costing_p) + break; - /* 2. Create the vector-load in the loop. */ - switch (alignment_support_scheme) - { - case dr_aligned: - case dr_unaligned_supported: + unsigned int misalign; + unsigned HOST_WIDE_INT align; + align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info)); + if (alignment_support_scheme == dr_aligned) + misalign = 0; + else if (misalignment == DR_MISALIGNMENT_UNKNOWN) { - if (costing_p) - break; - - unsigned int misalign; - unsigned HOST_WIDE_INT align; - align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info)); - if (alignment_support_scheme == dr_aligned) - misalign = 0; - else if (misalignment == DR_MISALIGNMENT_UNKNOWN) - { - align - = dr_alignment (vect_dr_behavior (vinfo, first_dr_info)); - misalign = 0; - } - else - misalign = misalignment; - if (dataref_offset == NULL_TREE - && TREE_CODE (dataref_ptr) == SSA_NAME) - set_ptr_info_alignment (get_ptr_info (dataref_ptr), align, - misalign); - align = least_bit_hwi (misalign | align); - - /* Compute IFN when LOOP_LENS or final_mask valid. */ - machine_mode vmode = TYPE_MODE (vectype); - machine_mode new_vmode = vmode; - internal_fn partial_ifn = IFN_LAST; - if (loop_lens) + align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info)); + misalign = 0; + } + else + misalign = misalignment; + if (dataref_offset == NULL_TREE + && TREE_CODE (dataref_ptr) == SSA_NAME) + set_ptr_info_alignment (get_ptr_info (dataref_ptr), align, + misalign); + align = least_bit_hwi (misalign | align); + + /* Compute IFN when LOOP_LENS or final_mask valid. */ + machine_mode vmode = TYPE_MODE (vectype); + machine_mode new_vmode = vmode; + internal_fn partial_ifn = IFN_LAST; + if (loop_lens) + { + opt_machine_mode new_ovmode + = get_len_load_store_mode (vmode, true, &partial_ifn); + new_vmode = new_ovmode.require (); + unsigned factor + = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode); + final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens, + vec_num, vectype, i, factor); + } + else if (final_mask) + { + if (!can_vec_mask_load_store_p (vmode, + TYPE_MODE + (TREE_TYPE (final_mask)), + true, &partial_ifn)) + gcc_unreachable (); + } + + if (partial_ifn == IFN_MASK_LEN_LOAD) + { + if (!final_len) { - opt_machine_mode new_ovmode - = get_len_load_store_mode (vmode, true, &partial_ifn); - new_vmode = new_ovmode.require (); - unsigned factor - = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode); - final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens, - vec_num * ncopies, vectype, - vec_num * j + i, factor); + /* Pass VF value to 'len' argument of + MASK_LEN_LOAD if LOOP_LENS is invalid. */ + final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype)); } - else if (final_mask) + if (!final_mask) { - if (!can_vec_mask_load_store_p ( - vmode, TYPE_MODE (TREE_TYPE (final_mask)), true, - &partial_ifn)) - gcc_unreachable (); + /* Pass all ones value to 'mask' argument of + MASK_LEN_LOAD if final_mask is invalid. */ + mask_vectype = truth_type_for (vectype); + final_mask = build_minus_one_cst (mask_vectype); } + } + if (final_len) + { + signed char biasval + = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); + bias = build_int_cst (intQI_type_node, biasval); + } - if (partial_ifn == IFN_MASK_LEN_LOAD) - { - if (!final_len) - { - /* Pass VF value to 'len' argument of - MASK_LEN_LOAD if LOOP_LENS is invalid. */ - final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype)); - } - if (!final_mask) - { - /* Pass all ones value to 'mask' argument of - MASK_LEN_LOAD if final_mask is invalid. */ - mask_vectype = truth_type_for (vectype); - final_mask = build_minus_one_cst (mask_vectype); - } - } - if (final_len) - { - signed char biasval - = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); + tree vec_els; - bias = build_int_cst (intQI_type_node, biasval); - } + if (final_len) + { + tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT); + gcall *call; - tree vec_els; + /* Need conversion if the vectype is punned by VnQI. */ + els_vectype = vectype; + if (vmode != new_vmode) + els_vectype + = build_vector_type_for_mode (unsigned_intQI_type_node, + new_vmode); + vec_els = vect_get_mask_load_else (maskload_elsval, + els_vectype); - if (final_len) - { - tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT); - gcall *call; - if (partial_ifn == IFN_MASK_LEN_LOAD) - { - vec_els = vect_get_mask_load_else - (maskload_elsval, vectype); - if (type_mode_padding_p - && maskload_elsval != MASK_LOAD_ELSE_ZERO) - need_zeroing = true; - call = gimple_build_call_internal (IFN_MASK_LEN_LOAD, - 6, dataref_ptr, ptr, - final_mask, vec_els, - final_len, bias); - } - else - call = gimple_build_call_internal (IFN_LEN_LOAD, 4, - dataref_ptr, ptr, - final_len, bias); - gimple_call_set_nothrow (call, true); - new_stmt = call; - data_ref = NULL_TREE; - - /* Need conversion if it's wrapped with VnQI. */ - if (vmode != new_vmode) - { - tree new_vtype = build_vector_type_for_mode ( - unsigned_intQI_type_node, new_vmode); - tree var - = vect_get_new_ssa_name (new_vtype, vect_simple_var); - gimple_set_lhs (call, var); - vect_finish_stmt_generation (vinfo, stmt_info, call, - gsi); - tree op = build1 (VIEW_CONVERT_EXPR, vectype, var); - new_stmt = gimple_build_assign (vec_dest, - VIEW_CONVERT_EXPR, op); - } - } - else if (final_mask) + if (partial_ifn == IFN_MASK_LEN_LOAD) { - tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT); - vec_els = vect_get_mask_load_else - (maskload_elsval, vectype); if (type_mode_padding_p && maskload_elsval != MASK_LOAD_ELSE_ZERO) need_zeroing = true; - gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4, - dataref_ptr, ptr, - final_mask, - vec_els); - gimple_call_set_nothrow (call, true); - new_stmt = call; - data_ref = NULL_TREE; + call = gimple_build_call_internal (IFN_MASK_LEN_LOAD, + 6, dataref_ptr, ptr, + final_mask, vec_els, + final_len, bias); } else + call = gimple_build_call_internal (IFN_LEN_LOAD, 5, + dataref_ptr, ptr, + vec_els, final_len, + bias); + gimple_call_set_nothrow (call, true); + new_stmt = call; + data_ref = NULL_TREE; + + /* Need conversion if it's wrapped with VnQI. */ + if (vmode != new_vmode) { - tree ltype = vectype; - tree new_vtype = NULL_TREE; - unsigned HOST_WIDE_INT gap = DR_GROUP_GAP (first_stmt_info); - unsigned HOST_WIDE_INT dr_size - = vect_get_scalar_dr_size (first_dr_info); - poly_int64 off = 0; - if (memory_access_type == VMAT_CONTIGUOUS_REVERSE) - off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size; - unsigned int vect_align - = vect_known_alignment_in_bytes (first_dr_info, vectype, - off); - /* Try to use a single smaller load when we are about - to load excess elements compared to the unrolled - scalar loop. */ - if (known_gt ((vec_num * j + i + 1) * nunits, - (group_size * vf - gap))) + tree new_vtype + = build_vector_type_for_mode (unsigned_intQI_type_node, + new_vmode); + tree var = vect_get_new_ssa_name (new_vtype, + vect_simple_var); + gimple_set_lhs (call, var); + vect_finish_stmt_generation (vinfo, stmt_info, call, + gsi); + tree op = build1 (VIEW_CONVERT_EXPR, vectype, var); + new_stmt = gimple_build_assign (vec_dest, + VIEW_CONVERT_EXPR, op); + } + } + else if (final_mask) + { + tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT); + vec_els = vect_get_mask_load_else (maskload_elsval, vectype); + if (type_mode_padding_p + && maskload_elsval != MASK_LOAD_ELSE_ZERO) + need_zeroing = true; + gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4, + dataref_ptr, ptr, + final_mask, + vec_els); + gimple_call_set_nothrow (call, true); + new_stmt = call; + data_ref = NULL_TREE; + } + else + { + tree ltype = vectype; + tree new_vtype = NULL_TREE; + unsigned HOST_WIDE_INT gap = DR_GROUP_GAP (first_stmt_info); + unsigned HOST_WIDE_INT dr_size + = vect_get_scalar_dr_size (first_dr_info); + poly_int64 off = 0; + if (memory_access_type == VMAT_CONTIGUOUS_REVERSE) + off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size; + unsigned int vect_align + = vect_known_alignment_in_bytes (first_dr_info, vectype, + off); + /* Try to use a single smaller load when we are about + to load excess elements compared to the unrolled + scalar loop. */ + if (known_gt ((i + 1) * nunits, + (group_size * vf - gap))) + { + poly_uint64 remain = ((group_size * vf - gap) - i * nunits); + if (known_ge ((i + 1) * nunits - (group_size * vf - gap), + nunits)) + /* DR will be unused. */ + ltype = NULL_TREE; + else if (known_ge (vect_align, + tree_to_poly_uint64 + (TYPE_SIZE_UNIT (vectype)))) + /* Aligned access to excess elements is OK if + at least one element is accessed in the + scalar loop. */ + ; + else if (known_gt (vect_align, + ((nunits - remain) * dr_size))) + /* Aligned access to the gap area when there's + at least one element in it is OK. */ + ; + else { - poly_uint64 remain = ((group_size * vf - gap) - - (vec_num * j + i) * nunits); - if (known_ge ((vec_num * j + i + 1) * nunits - - (group_size * vf - gap), nunits)) - /* DR will be unused. */ - ltype = NULL_TREE; - else if (known_ge (vect_align, - tree_to_poly_uint64 - (TYPE_SIZE_UNIT (vectype)))) - /* Aligned access to excess elements is OK if - at least one element is accessed in the - scalar loop. */ - ; - else if (known_gt (vect_align, - ((nunits - remain) * dr_size))) - /* Aligned access to the gap area when there's - at least one element in it is OK. */ - ; - else + /* remain should now be > 0 and < nunits. */ + unsigned num; + if (known_ne (remain, 0u) + && constant_multiple_p (nunits, remain, &num)) + { + tree ptype; + new_vtype + = vector_vector_composition_type (vectype, num, + &ptype); + if (new_vtype) + ltype = ptype; + } + /* Else use multiple loads or a masked load? */ + /* For loop vectorization we now should have + an alternate type or LOOP_VINFO_PEELING_FOR_GAPS + set. */ + if (loop_vinfo) + gcc_assert (new_vtype + || LOOP_VINFO_PEELING_FOR_GAPS + (loop_vinfo)); + /* But still reduce the access size to the next + required power-of-two so peeling a single + scalar iteration is sufficient. */ + unsigned HOST_WIDE_INT cremain; + if (remain.is_constant (&cremain)) { - /* remain should now be > 0 and < nunits. */ - unsigned num; - if (known_ne (remain, 0u) - && constant_multiple_p (nunits, remain, &num)) + unsigned HOST_WIDE_INT cpart_size + = 1 << ceil_log2 (cremain); + if (known_gt (nunits, cpart_size) + && constant_multiple_p (nunits, cpart_size, + &num)) { tree ptype; new_vtype @@ -12199,334 +11530,284 @@ vectorizable_load (vec_info *vinfo, if (new_vtype) ltype = ptype; } - /* Else use multiple loads or a masked load? */ - /* For loop vectorization we now should have - an alternate type or LOOP_VINFO_PEELING_FOR_GAPS - set. */ - if (loop_vinfo) - gcc_assert (new_vtype - || LOOP_VINFO_PEELING_FOR_GAPS - (loop_vinfo)); - /* But still reduce the access size to the next - required power-of-two so peeling a single - scalar iteration is sufficient. */ - unsigned HOST_WIDE_INT cremain; - if (remain.is_constant (&cremain)) - { - unsigned HOST_WIDE_INT cpart_size - = 1 << ceil_log2 (cremain); - if (known_gt (nunits, cpart_size) - && constant_multiple_p (nunits, cpart_size, - &num)) - { - tree ptype; - new_vtype - = vector_vector_composition_type (vectype, - num, - &ptype); - if (new_vtype) - ltype = ptype; - } - } } } - tree offset - = (dataref_offset ? dataref_offset - : build_int_cst (ref_type, 0)); - if (!ltype) + } + tree offset = (dataref_offset ? dataref_offset + : build_int_cst (ref_type, 0)); + if (!ltype) + ; + else if (ltype != vectype + && memory_access_type == VMAT_CONTIGUOUS_REVERSE) + { + poly_uint64 gap_offset + = (tree_to_poly_uint64 (TYPE_SIZE_UNIT (vectype)) + - tree_to_poly_uint64 (TYPE_SIZE_UNIT (ltype))); + tree gapcst = build_int_cstu (ref_type, gap_offset); + offset = size_binop (PLUS_EXPR, offset, gapcst); + } + if (ltype) + { + data_ref = fold_build2 (MEM_REF, ltype, + dataref_ptr, offset); + if (alignment_support_scheme == dr_aligned + && align >= TYPE_ALIGN_UNIT (ltype)) ; - else if (ltype != vectype - && memory_access_type == VMAT_CONTIGUOUS_REVERSE) + else + TREE_TYPE (data_ref) + = build_aligned_type (TREE_TYPE (data_ref), + align * BITS_PER_UNIT); + } + if (!ltype) + data_ref = build_constructor (vectype, NULL); + else if (ltype != vectype) + { + vect_copy_ref_info (data_ref, + DR_REF (first_dr_info->dr)); + tree tem = make_ssa_name (ltype); + new_stmt = gimple_build_assign (tem, data_ref); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, + gsi); + data_ref = NULL; + vec<constructor_elt, va_gc> *v; + /* We've computed 'num' above to statically two + or via constant_multiple_p. */ + unsigned num + = (exact_div (tree_to_poly_uint64 + (TYPE_SIZE_UNIT (vectype)), + tree_to_poly_uint64 + (TYPE_SIZE_UNIT (ltype))) + .to_constant ()); + vec_alloc (v, num); + if (memory_access_type == VMAT_CONTIGUOUS_REVERSE) { - poly_uint64 gap_offset - = (tree_to_poly_uint64 (TYPE_SIZE_UNIT (vectype)) - - tree_to_poly_uint64 (TYPE_SIZE_UNIT (ltype))); - tree gapcst = build_int_cstu (ref_type, gap_offset); - offset = size_binop (PLUS_EXPR, offset, gapcst); + while (--num) + CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, + build_zero_cst (ltype)); + CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem); } - if (ltype) + else { - data_ref - = fold_build2 (MEM_REF, ltype, dataref_ptr, offset); - if (alignment_support_scheme == dr_aligned) - ; - else - TREE_TYPE (data_ref) - = build_aligned_type (TREE_TYPE (data_ref), - align * BITS_PER_UNIT); + CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem); + while (--num) + CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, + build_zero_cst (ltype)); } - if (!ltype) - data_ref = build_constructor (vectype, NULL); - else if (ltype != vectype) + gcc_assert (new_vtype != NULL_TREE); + if (new_vtype == vectype) + new_stmt + = gimple_build_assign (vec_dest, + build_constructor (vectype, v)); + else { - vect_copy_ref_info (data_ref, - DR_REF (first_dr_info->dr)); - tree tem = make_ssa_name (ltype); - new_stmt = gimple_build_assign (tem, data_ref); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, - gsi); - data_ref = NULL; - vec<constructor_elt, va_gc> *v; - /* We've computed 'num' above to statically two - or via constant_multiple_p. */ - unsigned num - = (exact_div (tree_to_poly_uint64 - (TYPE_SIZE_UNIT (vectype)), - tree_to_poly_uint64 - (TYPE_SIZE_UNIT (ltype))) - .to_constant ()); - vec_alloc (v, num); - if (memory_access_type == VMAT_CONTIGUOUS_REVERSE) - { - while (--num) - CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, - build_zero_cst (ltype)); - CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem); - } - else - { - CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem); - while (--num) - CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, - build_zero_cst (ltype)); - } - gcc_assert (new_vtype != NULL_TREE); - if (new_vtype == vectype) - new_stmt = gimple_build_assign ( - vec_dest, build_constructor (vectype, v)); - else - { - tree new_vname = make_ssa_name (new_vtype); - new_stmt = gimple_build_assign ( - new_vname, build_constructor (new_vtype, v)); - vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); - new_stmt = gimple_build_assign ( - vec_dest, - build1 (VIEW_CONVERT_EXPR, vectype, new_vname)); - } + tree new_vname = make_ssa_name (new_vtype); + new_stmt + = gimple_build_assign (new_vname, + build_constructor (new_vtype, + v)); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); + new_stmt + = gimple_build_assign (vec_dest, + build1 (VIEW_CONVERT_EXPR, + vectype, new_vname)); } } - break; } - case dr_explicit_realign: - { - if (costing_p) - break; - tree ptr, bump; - - tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype)); - - if (compute_in_loop) - msq = vect_setup_realignment (vinfo, first_stmt_info, gsi, - &realignment_token, - dr_explicit_realign, - dataref_ptr, NULL); + break; + } + case dr_explicit_realign: + { + if (costing_p) + break; + tree ptr, bump; + + tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype)); + + if (compute_in_loop) + msq = vect_setup_realignment (vinfo, first_stmt_info, vectype, + gsi, &realignment_token, + dr_explicit_realign, + dataref_ptr, NULL); + + if (TREE_CODE (dataref_ptr) == SSA_NAME) + ptr = copy_ssa_name (dataref_ptr); + else + ptr = make_ssa_name (TREE_TYPE (dataref_ptr)); + // For explicit realign the target alignment should be + // known at compile time. + unsigned HOST_WIDE_INT align + = DR_TARGET_ALIGNMENT (first_dr_info).to_constant (); + new_stmt = gimple_build_assign (ptr, BIT_AND_EXPR, dataref_ptr, + build_int_cst + (TREE_TYPE (dataref_ptr), + -(HOST_WIDE_INT) align)); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + data_ref = build2 (MEM_REF, vectype, + ptr, build_int_cst (ref_type, 0)); + vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); + vec_dest = vect_create_destination_var (scalar_dest, vectype); + new_stmt = gimple_build_assign (vec_dest, data_ref); + new_temp = make_ssa_name (vec_dest, new_stmt); + gimple_assign_set_lhs (new_stmt, new_temp); + gimple_move_vops (new_stmt, stmt_info->stmt); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + msq = new_temp; + + bump = size_binop (MULT_EXPR, vs, TYPE_SIZE_UNIT (elem_type)); + bump = size_binop (MINUS_EXPR, bump, size_one_node); + ptr = bump_vector_ptr (vinfo, dataref_ptr, NULL, gsi, stmt_info, + bump); + new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR, ptr, + build_int_cst (TREE_TYPE (ptr), + -(HOST_WIDE_INT) align)); + if (TREE_CODE (ptr) == SSA_NAME) + ptr = copy_ssa_name (ptr, new_stmt); + else + ptr = make_ssa_name (TREE_TYPE (ptr), new_stmt); + gimple_assign_set_lhs (new_stmt, ptr); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + data_ref = build2 (MEM_REF, vectype, + ptr, build_int_cst (ref_type, 0)); + break; + } + case dr_explicit_realign_optimized: + { + if (costing_p) + break; + if (TREE_CODE (dataref_ptr) == SSA_NAME) + new_temp = copy_ssa_name (dataref_ptr); + else + new_temp = make_ssa_name (TREE_TYPE (dataref_ptr)); + // We should only be doing this if we know the target + // alignment at compile time. + unsigned HOST_WIDE_INT align + = DR_TARGET_ALIGNMENT (first_dr_info).to_constant (); + new_stmt = gimple_build_assign (new_temp, BIT_AND_EXPR, dataref_ptr, + build_int_cst (TREE_TYPE (dataref_ptr), + -(HOST_WIDE_INT) align)); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + data_ref = build2 (MEM_REF, vectype, new_temp, + build_int_cst (ref_type, 0)); + break; + } + default: + gcc_unreachable (); + } - if (TREE_CODE (dataref_ptr) == SSA_NAME) - ptr = copy_ssa_name (dataref_ptr); - else - ptr = make_ssa_name (TREE_TYPE (dataref_ptr)); - // For explicit realign the target alignment should be - // known at compile time. - unsigned HOST_WIDE_INT align - = DR_TARGET_ALIGNMENT (first_dr_info).to_constant (); - new_stmt = gimple_build_assign ( - ptr, BIT_AND_EXPR, dataref_ptr, - build_int_cst (TREE_TYPE (dataref_ptr), - -(HOST_WIDE_INT) align)); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - data_ref - = build2 (MEM_REF, vectype, ptr, build_int_cst (ref_type, 0)); - vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); - vec_dest = vect_create_destination_var (scalar_dest, vectype); - new_stmt = gimple_build_assign (vec_dest, data_ref); - new_temp = make_ssa_name (vec_dest, new_stmt); - gimple_assign_set_lhs (new_stmt, new_temp); - gimple_move_vops (new_stmt, stmt_info->stmt); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - msq = new_temp; - - bump = size_binop (MULT_EXPR, vs, TYPE_SIZE_UNIT (elem_type)); - bump = size_binop (MINUS_EXPR, bump, size_one_node); - ptr = bump_vector_ptr (vinfo, dataref_ptr, NULL, gsi, stmt_info, - bump); - new_stmt = gimple_build_assign ( - NULL_TREE, BIT_AND_EXPR, ptr, - build_int_cst (TREE_TYPE (ptr), -(HOST_WIDE_INT) align)); - if (TREE_CODE (ptr) == SSA_NAME) - ptr = copy_ssa_name (ptr, new_stmt); - else - ptr = make_ssa_name (TREE_TYPE (ptr), new_stmt); - gimple_assign_set_lhs (new_stmt, ptr); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - data_ref - = build2 (MEM_REF, vectype, ptr, build_int_cst (ref_type, 0)); - break; - } - case dr_explicit_realign_optimized: - { - if (costing_p) - break; - if (TREE_CODE (dataref_ptr) == SSA_NAME) - new_temp = copy_ssa_name (dataref_ptr); - else - new_temp = make_ssa_name (TREE_TYPE (dataref_ptr)); - // We should only be doing this if we know the target - // alignment at compile time. - unsigned HOST_WIDE_INT align - = DR_TARGET_ALIGNMENT (first_dr_info).to_constant (); - new_stmt = gimple_build_assign ( - new_temp, BIT_AND_EXPR, dataref_ptr, - build_int_cst (TREE_TYPE (dataref_ptr), - -(HOST_WIDE_INT) align)); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - data_ref = build2 (MEM_REF, vectype, new_temp, - build_int_cst (ref_type, 0)); - break; - } - default: - gcc_unreachable (); + /* One common place to cost the above vect load for different + alignment support schemes. */ + if (costing_p) + { + /* For the prologue cost for realign, + we only need to count it once for the whole group. */ + bool first_stmt_info_p = first_stmt_info == stmt_info; + bool add_realign_cost = first_stmt_info_p && i == 0; + if (memory_access_type == VMAT_CONTIGUOUS + || memory_access_type == VMAT_CONTIGUOUS_REVERSE) + { + /* Leave realign cases alone to keep them simple. */ + if (alignment_support_scheme == dr_explicit_realign_optimized + || alignment_support_scheme == dr_explicit_realign) + vect_get_load_cost (vinfo, stmt_info, slp_node, 1, + alignment_support_scheme, misalignment, + add_realign_cost, &inside_cost, + &prologue_cost, cost_vec, cost_vec, + true); + else + n_adjacent_loads++; } - - /* One common place to cost the above vect load for different - alignment support schemes. */ - if (costing_p) + } + else + { + vec_dest = vect_create_destination_var (scalar_dest, vectype); + /* DATA_REF is null if we've already built the statement. */ + if (data_ref) { - /* For VMAT_CONTIGUOUS_PERMUTE if it's grouped load, we - only need to take care of the first stmt, whose - stmt_info is first_stmt_info, vec_num iterating on it - will cover the cost for the remaining, it's consistent - with transforming. For the prologue cost for realign, - we only need to count it once for the whole group. */ - bool first_stmt_info_p = first_stmt_info == stmt_info; - bool add_realign_cost = first_stmt_info_p && i == 0; - if (memory_access_type == VMAT_CONTIGUOUS - || memory_access_type == VMAT_CONTIGUOUS_REVERSE - || (memory_access_type == VMAT_CONTIGUOUS_PERMUTE - && (!grouped_load || first_stmt_info_p))) - { - /* Leave realign cases alone to keep them simple. */ - if (alignment_support_scheme == dr_explicit_realign_optimized - || alignment_support_scheme == dr_explicit_realign) - vect_get_load_cost (vinfo, stmt_info, slp_node, 1, - alignment_support_scheme, misalignment, - add_realign_cost, &inside_cost, - &prologue_cost, cost_vec, cost_vec, - true); - else - n_adjacent_loads++; - } + vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); + new_stmt = gimple_build_assign (vec_dest, data_ref); } - else - { - vec_dest = vect_create_destination_var (scalar_dest, vectype); - /* DATA_REF is null if we've already built the statement. */ - if (data_ref) - { - vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); - new_stmt = gimple_build_assign (vec_dest, data_ref); - } - new_temp = need_zeroing - ? make_ssa_name (vectype) - : make_ssa_name (vec_dest, new_stmt); - gimple_set_lhs (new_stmt, new_temp); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_temp = (need_zeroing + ? make_ssa_name (vectype) + : make_ssa_name (vec_dest, new_stmt)); + gimple_set_lhs (new_stmt, new_temp); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - /* If we need to explicitly zero inactive elements emit a - VEC_COND_EXPR that does so. */ - if (need_zeroing) - { - vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO, - vectype); + /* If we need to explicitly zero inactive elements emit a + VEC_COND_EXPR that does so. */ + if (need_zeroing) + { + vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO, + vectype); - tree new_temp2 = make_ssa_name (vec_dest, new_stmt); - new_stmt - = gimple_build_assign (new_temp2, VEC_COND_EXPR, - final_mask, new_temp, vec_els); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, - gsi); - new_temp = new_temp2; - } + tree new_temp2 = make_ssa_name (vec_dest, new_stmt); + new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR, + final_mask, new_temp, vec_els); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, + gsi); + new_temp = new_temp2; } + } - /* 3. Handle explicit realignment if necessary/supported. - Create in loop: - vec_dest = realign_load (msq, lsq, realignment_token) */ - if (!costing_p - && (alignment_support_scheme == dr_explicit_realign_optimized - || alignment_support_scheme == dr_explicit_realign)) - { - lsq = gimple_assign_lhs (new_stmt); - if (!realignment_token) - realignment_token = dataref_ptr; - vec_dest = vect_create_destination_var (scalar_dest, vectype); - new_stmt = gimple_build_assign (vec_dest, REALIGN_LOAD_EXPR, msq, - lsq, realignment_token); - new_temp = make_ssa_name (vec_dest, new_stmt); - gimple_assign_set_lhs (new_stmt, new_temp); - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + /* 3. Handle explicit realignment if necessary/supported. + Create in loop: + vec_dest = realign_load (msq, lsq, realignment_token) */ + if (!costing_p + && (alignment_support_scheme == dr_explicit_realign_optimized + || alignment_support_scheme == dr_explicit_realign)) + { + lsq = gimple_assign_lhs (new_stmt); + if (!realignment_token) + realignment_token = dataref_ptr; + vec_dest = vect_create_destination_var (scalar_dest, vectype); + new_stmt = gimple_build_assign (vec_dest, REALIGN_LOAD_EXPR, msq, + lsq, realignment_token); + new_temp = make_ssa_name (vec_dest, new_stmt); + gimple_assign_set_lhs (new_stmt, new_temp); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - if (alignment_support_scheme == dr_explicit_realign_optimized) - { - gcc_assert (phi); - if (i == vec_num - 1 && j == ncopies - 1) - add_phi_arg (phi, lsq, loop_latch_edge (containing_loop), - UNKNOWN_LOCATION); - msq = lsq; - } + if (alignment_support_scheme == dr_explicit_realign_optimized) + { + gcc_assert (phi); + if (i == vec_num - 1) + add_phi_arg (phi, lsq, loop_latch_edge (containing_loop), + UNKNOWN_LOCATION); + msq = lsq; } + } - if (memory_access_type == VMAT_CONTIGUOUS_REVERSE) + if (memory_access_type == VMAT_CONTIGUOUS_REVERSE) + { + if (costing_p) + inside_cost = record_stmt_cost (cost_vec, 1, vec_perm, + slp_node, 0, vect_body); + else { - if (costing_p) - inside_cost = record_stmt_cost (cost_vec, 1, vec_perm, - stmt_info, slp_node, 0, - vect_body); - else - { - tree perm_mask = perm_mask_for_reverse (vectype); - new_temp = permute_vec_elements (vinfo, new_temp, new_temp, - perm_mask, stmt_info, gsi); - new_stmt = SSA_NAME_DEF_STMT (new_temp); - } + tree perm_mask = perm_mask_for_reverse (vectype); + new_temp = permute_vec_elements (vinfo, new_temp, new_temp, + perm_mask, stmt_info, gsi); + new_stmt = SSA_NAME_DEF_STMT (new_temp); } + } - /* Collect vector loads and later create their permutation in - vect_transform_grouped_load (). */ - if (!costing_p && (grouped_load || slp_perm)) - dr_chain.quick_push (new_temp); + /* Collect vector loads and later create their permutation in + vect_transform_slp_perm_load. */ + if (!costing_p && (grouped_load || ls.slp_perm)) + dr_chain.quick_push (new_temp); - /* Store vector loads in the corresponding SLP_NODE. */ - if (!costing_p && slp && !slp_perm) - slp_node->push_vec_def (new_stmt); - - /* With SLP permutation we load the gaps as well, without - we need to skip the gaps after we manage to fully load - all elements. group_gap_adj is DR_GROUP_SIZE here. */ - group_elt += nunits; - if (!costing_p - && maybe_ne (group_gap_adj, 0U) - && !slp_perm - && known_eq (group_elt, group_size - group_gap_adj)) - { - poly_wide_int bump_val - = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj); - if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) - == -1) - bump_val = -bump_val; - tree bump = wide_int_to_tree (sizetype, bump_val); - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, - stmt_info, bump); - group_elt = 0; - } - } - /* Bump the vector pointer to account for a gap or for excess - elements loaded for a permuted SLP load. */ + /* Store vector loads in the corresponding SLP_NODE. */ + if (!costing_p && !ls.slp_perm) + slp_node->push_vec_def (new_stmt); + + /* With SLP permutation we load the gaps as well, without + we need to skip the gaps after we manage to fully load + all elements. group_gap_adj is DR_GROUP_SIZE here. */ + group_elt += nunits; if (!costing_p && maybe_ne (group_gap_adj, 0U) - && slp_perm) + && !ls.slp_perm + && known_eq (group_elt, group_size - group_gap_adj)) { poly_wide_int bump_val = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj); @@ -12535,80 +11816,51 @@ vectorizable_load (vec_info *vinfo, tree bump = wide_int_to_tree (sizetype, bump_val); dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, stmt_info, bump); + group_elt = 0; } + } + /* Bump the vector pointer to account for a gap or for excess + elements loaded for a permuted SLP load. */ + if (!costing_p + && maybe_ne (group_gap_adj, 0U) + && ls.slp_perm) + { + poly_wide_int bump_val + = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj); + if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1) + bump_val = -bump_val; + tree bump = wide_int_to_tree (sizetype, bump_val); + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, + stmt_info, bump); + } - if (slp && !slp_perm) - continue; - - if (slp_perm) + if (ls.slp_perm) + { + /* For SLP we know we've seen all possible uses of dr_chain so + direct vect_transform_slp_perm_load to DCE the unused parts. + ??? This is a hack to prevent compile-time issues as seen + in PR101120 and friends. */ + if (costing_p) { - unsigned n_perms; - /* For SLP we know we've seen all possible uses of dr_chain so - direct vect_transform_slp_perm_load to DCE the unused parts. - ??? This is a hack to prevent compile-time issues as seen - in PR101120 and friends. */ - if (costing_p) - { - vect_transform_slp_perm_load (vinfo, slp_node, vNULL, nullptr, vf, - true, &n_perms, nullptr); - inside_cost = record_stmt_cost (cost_vec, n_perms, vec_perm, - stmt_info, slp_node, 0, - vect_body); - } - else - { - bool ok = vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, - gsi, vf, false, &n_perms, - nullptr, true); - gcc_assert (ok); - } + gcc_assert (ls.n_perms != -1U); + if (ls.n_perms != 0) + inside_cost = record_stmt_cost (cost_vec, ls.n_perms, vec_perm, + slp_node, 0, vect_body); } else { - if (grouped_load) - { - gcc_assert (memory_access_type == VMAT_CONTIGUOUS_PERMUTE); - /* We assume that the cost of a single load-lanes instruction - is equivalent to the cost of DR_GROUP_SIZE separate loads. - If a grouped access is instead being provided by a - load-and-permute operation, include the cost of the - permutes. */ - if (costing_p && first_stmt_info == stmt_info) - { - /* Uses an even and odd extract operations or shuffle - operations for each needed permute. */ - int group_size = DR_GROUP_SIZE (first_stmt_info); - int nstmts = ceil_log2 (group_size) * group_size; - inside_cost += record_stmt_cost (cost_vec, nstmts, vec_perm, - stmt_info, slp_node, 0, - vect_body); - - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "vect_model_load_cost: " - "strided group_size = %d .\n", - group_size); - } - else if (!costing_p) - { - vect_transform_grouped_load (vinfo, stmt_info, dr_chain, - group_size, gsi); - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - } - } - else if (!costing_p) - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + unsigned n_perms2; + bool ok = vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, + gsi, vf, false, &n_perms2, + nullptr, true); + gcc_assert (ok && ls.n_perms == n_perms2); } - dr_chain.release (); } - if (!slp && !costing_p) - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; if (costing_p) { gcc_assert (memory_access_type == VMAT_CONTIGUOUS - || memory_access_type == VMAT_CONTIGUOUS_REVERSE - || memory_access_type == VMAT_CONTIGUOUS_PERMUTE); + || memory_access_type == VMAT_CONTIGUOUS_REVERSE); if (n_adjacent_loads > 0) vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads, alignment_support_scheme, misalignment, false, @@ -12638,7 +11890,7 @@ vectorizable_load (vec_info *vinfo, condition operands are supportable using vec_is_simple_use. */ static bool -vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info, +vect_is_simple_cond (tree cond, vec_info *vinfo, slp_tree slp_node, tree *comp_vectype, enum vect_def_type *dts, tree vectype) { @@ -12650,7 +11902,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info, if (TREE_CODE (cond) == SSA_NAME && VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond))) { - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 0, &cond, + if (!vect_is_simple_use (vinfo, slp_node, 0, &cond, &slp_op, &dts[0], comp_vectype) || !*comp_vectype || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype)) @@ -12666,7 +11918,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info, if (TREE_CODE (lhs) == SSA_NAME) { - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 0, + if (!vect_is_simple_use (vinfo, slp_node, 0, &lhs, &slp_op, &dts[0], &vectype1)) return false; } @@ -12678,7 +11930,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info, if (TREE_CODE (rhs) == SSA_NAME) { - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1, + if (!vect_is_simple_use (vinfo, slp_node, 1, &rhs, &slp_op, &dts[1], &vectype2)) return false; } @@ -12720,9 +11972,9 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info, /* vectorizable_condition. Check if STMT_INFO is conditional modify expression that can be vectorized. - If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized - stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it - at GSI. + If COST_VEC is passed, calculate costs but don't change anything, + otherwise, vectorize STMT_INFO: create a vectorized stmt using + VEC_COND_EXPR to replace it, and insert it at GSI. When STMT_INFO is vectorized as a nested cycle, for_reduction is true. @@ -12731,7 +11983,6 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info, static bool vectorizable_condition (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree scalar_dest = NULL_TREE; @@ -12747,9 +11998,6 @@ vectorizable_condition (vec_info *vinfo, enum vect_def_type dts[4] = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type}; - int ndts = 4; - int ncopies; - int vec_num; enum tree_code code, cond_code, bitop1 = NOP_EXPR, bitop2 = NOP_EXPR; int i; bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo); @@ -12772,20 +12020,24 @@ vectorizable_condition (vec_info *vinfo, if (code != COND_EXPR) return false; - stmt_vec_info reduc_info = NULL; - int reduc_index = -1; + int reduc_index = SLP_TREE_REDUC_IDX (slp_node); vect_reduction_type reduction_type = TREE_CODE_REDUCTION; - bool for_reduction - = STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)) != NULL; + bool nested_cycle_p = false; + bool for_reduction = vect_is_reduction (stmt_info); if (for_reduction) { - if (slp_node && SLP_TREE_LANES (slp_node) > 1) + if (SLP_TREE_LANES (slp_node) > 1) return false; - reduc_info = info_for_reduction (vinfo, stmt_info); - reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info); - reduc_index = STMT_VINFO_REDUC_IDX (stmt_info); - gcc_assert (reduction_type != EXTRACT_LAST_REDUCTION - || reduc_index != -1); + /* ??? With a reduction path we do not get at the reduction info from + every stmt, use the conservative default setting then. */ + if (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info))) + { + vect_reduc_info reduc_info + = info_for_reduction (loop_vinfo, slp_node); + reduction_type = VECT_REDUC_INFO_TYPE (reduc_info); + nested_cycle_p = nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), + stmt_info); + } } else { @@ -12793,38 +12045,25 @@ vectorizable_condition (vec_info *vinfo, return false; } - tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vectype = SLP_TREE_VECTYPE (slp_node); tree vectype1 = NULL_TREE, vectype2 = NULL_TREE; - if (slp_node) - { - ncopies = 1; - vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); - } - else - { - ncopies = vect_get_num_copies (loop_vinfo, vectype); - vec_num = 1; - } - - gcc_assert (ncopies >= 1); - if (for_reduction && ncopies > 1) - return false; /* FORNOW */ + int vec_num = vect_get_num_copies (vinfo, slp_node); cond_expr = gimple_assign_rhs1 (stmt); gcc_assert (! COMPARISON_CLASS_P (cond_expr)); - if (!vect_is_simple_cond (cond_expr, vinfo, stmt_info, slp_node, + if (!vect_is_simple_cond (cond_expr, vinfo, slp_node, &comp_vectype, &dts[0], vectype) || !comp_vectype) return false; unsigned op_adjust = COMPARISON_CLASS_P (cond_expr) ? 1 : 0; slp_tree then_slp_node, else_slp_node; - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1 + op_adjust, + if (!vect_is_simple_use (vinfo, slp_node, 1 + op_adjust, &then_clause, &then_slp_node, &dts[2], &vectype1)) return false; - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 2 + op_adjust, + if (!vect_is_simple_use (vinfo, slp_node, 2 + op_adjust, &else_clause, &else_slp_node, &dts[3], &vectype2)) return false; @@ -12872,9 +12111,8 @@ vectorizable_condition (vec_info *vinfo, } /* ??? The vectorized operand query below doesn't allow swapping this way for SLP. */ - if (slp_node) - return false; - std::swap (then_clause, else_clause); + return false; + /* std::swap (then_clause, else_clause); */ } if (!masked && VECTOR_BOOLEAN_TYPE_P (comp_vectype)) @@ -12928,7 +12166,7 @@ vectorizable_condition (vec_info *vinfo, return false; } - if (!vec_stmt) + if (cost_vec) { if (bitop1 != NOP_EXPR) { @@ -12959,14 +12197,13 @@ vectorizable_condition (vec_info *vinfo, || !expand_vec_cond_expr_p (vectype, vec_cmp_type)))) return false; - if (slp_node - && (!vect_maybe_update_slp_op_vectype - (SLP_TREE_CHILDREN (slp_node)[0], comp_vectype) - || (op_adjust == 1 - && !vect_maybe_update_slp_op_vectype - (SLP_TREE_CHILDREN (slp_node)[1], comp_vectype)) - || !vect_maybe_update_slp_op_vectype (then_slp_node, vectype) - || !vect_maybe_update_slp_op_vectype (else_slp_node, vectype))) + if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0], + comp_vectype) + || (op_adjust == 1 + && !vect_maybe_update_slp_op_vectype + (SLP_TREE_CHILDREN (slp_node)[1], comp_vectype)) + || !vect_maybe_update_slp_op_vectype (then_slp_node, vectype) + || !vect_maybe_update_slp_op_vectype (else_slp_node, vectype)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -12983,14 +12220,14 @@ vectorizable_condition (vec_info *vinfo, vectype, OPTIMIZE_FOR_SPEED)) vect_record_loop_len (loop_vinfo, &LOOP_VINFO_LENS (loop_vinfo), - ncopies * vec_num, vectype, 1); + vec_num, vectype, 1); else vect_record_loop_mask (loop_vinfo, &LOOP_VINFO_MASKS (loop_vinfo), - ncopies * vec_num, vectype, NULL); + vec_num, vectype, NULL); } /* Extra inactive lanes should be safe for vect_nested_cycle. */ - else if (STMT_VINFO_DEF_TYPE (reduc_info) != vect_nested_cycle) + else if (!nested_cycle_p) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -13000,9 +12237,8 @@ vectorizable_condition (vec_info *vinfo, } } - STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type; - vect_model_simple_cost (vinfo, stmt_info, ncopies, dts, ndts, slp_node, - cost_vec, kind); + SLP_TREE_TYPE (slp_node) = condition_vec_info_type; + vect_model_simple_cost (vinfo, 1, slp_node, cost_vec, kind); return true; } @@ -13031,7 +12267,7 @@ vectorizable_condition (vec_info *vinfo, masks = &LOOP_VINFO_MASKS (loop_vinfo); else { - scalar_cond_masked_key cond (cond_expr, ncopies); + scalar_cond_masked_key cond (cond_expr, 1); if (loop_vinfo->scalar_cond_masked_set.contains (cond)) masks = &LOOP_VINFO_MASKS (loop_vinfo); else @@ -13066,18 +12302,18 @@ vectorizable_condition (vec_info *vinfo, /* Handle cond expr. */ if (masked) - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, - cond_expr, comp_vectype, &vec_oprnds0, - then_clause, vectype, &vec_oprnds2, + vect_get_vec_defs (vinfo, slp_node, + cond_expr, &vec_oprnds0, + then_clause, &vec_oprnds2, reduction_type != EXTRACT_LAST_REDUCTION - ? else_clause : NULL, vectype, &vec_oprnds3); + ? else_clause : NULL, &vec_oprnds3); else - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, - cond_expr0, comp_vectype, &vec_oprnds0, - cond_expr1, comp_vectype, &vec_oprnds1, - then_clause, vectype, &vec_oprnds2, + vect_get_vec_defs (vinfo, slp_node, + cond_expr0, &vec_oprnds0, + cond_expr1, &vec_oprnds1, + then_clause, &vec_oprnds2, reduction_type != EXTRACT_LAST_REDUCTION - ? else_clause : NULL, vectype, &vec_oprnds3); + ? else_clause : NULL, &vec_oprnds3); if (reduction_type == EXTRACT_LAST_REDUCTION) vec_else_clause = else_clause; @@ -13194,7 +12430,7 @@ vectorizable_condition (vec_info *vinfo, if (lens) { len = vect_get_loop_len (loop_vinfo, gsi, lens, - vec_num * ncopies, vectype, i, 1); + vec_num, vectype, i, 1); signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); bias = build_int_cst (intQI_type_node, biasval); @@ -13208,7 +12444,7 @@ vectorizable_condition (vec_info *vinfo, if (masks) { tree loop_mask - = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num * ncopies, + = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num, vectype, i); tree tmp2 = make_ssa_name (vec_cmp_type); gassign *g @@ -13260,15 +12496,9 @@ vectorizable_condition (vec_info *vinfo, vec_then_clause, vec_else_clause); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } - if (slp_node) - slp_node->push_vec_def (new_stmt); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + slp_node->push_vec_def (new_stmt); } - if (!slp_node) - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - vec_oprnds0.release (); vec_oprnds1.release (); vec_oprnds2.release (); @@ -13280,26 +12510,24 @@ vectorizable_condition (vec_info *vinfo, /* Helper of vectorizable_comparison. Check if STMT_INFO is comparison expression CODE that can be vectorized. - If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized - comparison, put it in VEC_STMT, and insert it at GSI. + If COST_VEC is passed, calculate costs but don't change anything, + otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert + it at GSI. Return true if STMT_INFO is vectorizable in this way. */ static bool vectorizable_comparison_1 (vec_info *vinfo, tree vectype, stmt_vec_info stmt_info, tree_code code, - gimple_stmt_iterator *gsi, gimple **vec_stmt, + gimple_stmt_iterator *gsi, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree lhs, rhs1, rhs2; tree vectype1 = NULL_TREE, vectype2 = NULL_TREE; tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE; tree new_temp; - loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type}; - int ndts = 2; poly_uint64 nunits; - int ncopies; enum tree_code bitop1 = NOP_EXPR, bitop2 = NOP_EXPR; int i; bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo); @@ -13317,22 +12545,15 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype, mask_type = vectype; nunits = TYPE_VECTOR_SUBPARTS (vectype); - if (slp_node) - ncopies = 1; - else - ncopies = vect_get_num_copies (loop_vinfo, vectype); - - gcc_assert (ncopies >= 1); - if (TREE_CODE_CLASS (code) != tcc_comparison) return false; slp_tree slp_rhs1, slp_rhs2; - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, + if (!vect_is_simple_use (vinfo, slp_node, 0, &rhs1, &slp_rhs1, &dts[0], &vectype1)) return false; - if (!vect_is_simple_use (vinfo, stmt_info, slp_node, + if (!vect_is_simple_use (vinfo, slp_node, 1, &rhs2, &slp_rhs2, &dts[1], &vectype2)) return false; @@ -13398,7 +12619,7 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype, } } - if (!vec_stmt) + if (cost_vec) { if (bitop1 == NOP_EXPR) { @@ -13423,9 +12644,8 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype, } /* Put types on constant and invariant SLP children. */ - if (slp_node - && (!vect_maybe_update_slp_op_vectype (slp_rhs1, vectype) - || !vect_maybe_update_slp_op_vectype (slp_rhs2, vectype))) + if (!vect_maybe_update_slp_op_vectype (slp_rhs1, vectype) + || !vect_maybe_update_slp_op_vectype (slp_rhs2, vectype)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -13433,9 +12653,8 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype, return false; } - vect_model_simple_cost (vinfo, stmt_info, - ncopies * (1 + (bitop2 != NOP_EXPR)), - dts, ndts, slp_node, cost_vec); + vect_model_simple_cost (vinfo, 1 + (bitop2 != NOP_EXPR), + slp_node, cost_vec); return true; } @@ -13446,9 +12665,7 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype, if (lhs) mask = vect_create_destination_var (lhs, mask_type); - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, - rhs1, vectype, &vec_oprnds0, - rhs2, vectype, &vec_oprnds1); + vect_get_vec_defs (vinfo, slp_node, rhs1, &vec_oprnds0, rhs2, &vec_oprnds1); if (swap_p) std::swap (vec_oprnds0, vec_oprnds1); @@ -13487,15 +12704,9 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype, vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } } - if (slp_node) - slp_node->push_vec_def (new_stmt); - else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + slp_node->push_vec_def (new_stmt); } - if (!slp_node) - *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - vec_oprnds0.release (); vec_oprnds1.release (); @@ -13505,15 +12716,15 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype, /* vectorizable_comparison. Check if STMT_INFO is comparison expression that can be vectorized. - If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized - comparison, put it in VEC_STMT, and insert it at GSI. + If COST_VEC is passed, calculate costs but don't change anything, + otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert + it at GSI. Return true if STMT_INFO is vectorizable in this way. */ static bool vectorizable_comparison (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo); @@ -13529,28 +12740,59 @@ vectorizable_comparison (vec_info *vinfo, return false; enum tree_code code = gimple_assign_rhs_code (stmt); - tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vectype = SLP_TREE_VECTYPE (slp_node); if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi, - vec_stmt, slp_node, cost_vec)) + slp_node, cost_vec)) return false; - if (!vec_stmt) - STMT_VINFO_TYPE (stmt_info) = comparison_vec_info_type; + if (cost_vec) + SLP_TREE_TYPE (slp_node) = comparison_vec_info_type; return true; } +/* Check to see if the target supports any of the compare and branch optabs for + vectors with MODE as these would be required when expanding. */ +static bool +supports_vector_compare_and_branch (loop_vec_info loop_vinfo, machine_mode mode) +{ + bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo); + bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo); + + /* The vectorizer only produces vec_cbranch_any_optab directly. So only + check for support for that or vec_cbranch_any_optab when masked. + We can't produce vcond_cbranch_any directly from the vectorizer as we + want to keep gimple_cond as the GIMPLE representation. But we'll fold + it in expand. For that reason we require a backend to support the + unconditional vector cbranch optab if they support the conditional one, + which is just an optimization on the unconditional one. */ + if (masked_loop_p + && direct_optab_handler (cond_vec_cbranch_any_optab, mode) + != CODE_FOR_nothing) + return true; + else if (len_loop_p + && direct_optab_handler (cond_len_vec_cbranch_any_optab, mode) + != CODE_FOR_nothing) + return true; + else if (!masked_loop_p && !len_loop_p + && direct_optab_handler (vec_cbranch_any_optab, mode) + != CODE_FOR_nothing) + return true; + + /* The target can implement cbranch to distinguish between boolean vector + types and data types if they don't have a different mode for both. */ + return direct_optab_handler (cbranch_optab, mode) != CODE_FOR_nothing; +} + /* Check to see if the current early break given in STMT_INFO is valid for vectorization. */ bool -vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info, - gimple_stmt_iterator *gsi, gimple **vec_stmt, +vectorizable_early_exit (loop_vec_info loop_vinfo, stmt_vec_info stmt_info, + gimple_stmt_iterator *gsi, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { - loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); - if (!loop_vinfo - || !is_a <gcond *> (STMT_VINFO_STMT (stmt_info))) + if (!is_a <gcond *> (STMT_VINFO_STMT (stmt_info))) return false; if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_condition_def) @@ -13563,48 +12805,17 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info, auto code = gimple_cond_code (STMT_VINFO_STMT (stmt_info)); - tree vectype = NULL_TREE; - slp_tree slp_op0; - tree op0; - enum vect_def_type dt0; - - /* Early break gcond kind SLP trees can be root only and have no children, - for instance in the case where the argument is an external. If that's - the case there is no operand to analyse use of. */ - if ((!slp_node || !SLP_TREE_CHILDREN (slp_node).is_empty ()) - && !vect_is_simple_use (vinfo, stmt_info, slp_node, 0, &op0, &slp_op0, &dt0, - &vectype)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "use not simple.\n"); - return false; - } - /* For SLP we don't want to use the type of the operands of the SLP node, when vectorizing using SLP slp_node will be the children of the gcond and we want to use the type of the direct children which since the gcond is root will be the current node, rather than a child node as vect_is_simple_use assumes. */ - if (slp_node) - vectype = SLP_TREE_VECTYPE (slp_node); - + tree vectype = SLP_TREE_VECTYPE (slp_node); if (!vectype) return false; machine_mode mode = TYPE_MODE (vectype); - int ncopies, vec_num; - - if (slp_node) - { - ncopies = 1; - vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); - } - else - { - ncopies = vect_get_num_copies (loop_vinfo, vectype); - vec_num = 1; - } + int vec_num = vect_get_num_copies (loop_vinfo, slp_node); vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo); vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo); @@ -13616,7 +12827,7 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info, gimple *orig_stmt = STMT_VINFO_STMT (vect_orig_stmt (stmt_info)); gcond *cond_stmt = as_a <gcond *>(orig_stmt); - tree cst = build_zero_cst (vectype); + tree vectype_out = vectype; auto bb = gimple_bb (cond_stmt); edge exit_true_edge = EDGE_SUCC (bb, 0); if (exit_true_edge->flags & EDGE_FALSE_VALUE) @@ -13633,44 +12844,57 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info, bool flipped = flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo), exit_true_edge->dest); - /* Analyze only. */ - if (!vec_stmt) + /* See if we support ADDHN and use that for the reduction. */ + internal_fn ifn = IFN_VEC_TRUNC_ADD_HIGH; + bool addhn_supported_p + = direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_BOTH); + tree narrow_type = NULL_TREE; + if (addhn_supported_p) { - if (direct_optab_handler (cbranch_optab, mode) == CODE_FOR_nothing) + /* Calculate the narrowing type for the result. */ + auto halfprec = TYPE_PRECISION (TREE_TYPE (vectype)) / 2; + auto unsignedp = TYPE_UNSIGNED (TREE_TYPE (vectype)); + tree itype = build_nonstandard_integer_type (halfprec, unsignedp); + tree tmp_type = build_vector_type (itype, TYPE_VECTOR_SUBPARTS (vectype)); + narrow_type = truth_type_for (tmp_type); + + if (!supports_vector_compare_and_branch (loop_vinfo, + TYPE_MODE (narrow_type))) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "can't vectorize early exit because the " - "target doesn't support flag setting vector " - "comparisons.\n"); - return false; + "can't use ADDHN reduction because cbranch for " + "the narrowed type is not supported by the " + "target.\n"); + addhn_supported_p = false; } + } - if (ncopies > 1 - && direct_optab_handler (ior_optab, mode) == CODE_FOR_nothing) + /* Analyze only. */ + if (cost_vec) + { + if (!addhn_supported_p + && !supports_vector_compare_and_branch (loop_vinfo, mode)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, "can't vectorize early exit because the " - "target does not support boolean vector IOR " - "for type %T.\n", - vectype); + "target doesn't support flag setting vector " + "comparisons.\n"); return false; } - if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi, - vec_stmt, slp_node, cost_vec)) + if (!vectorizable_comparison_1 (loop_vinfo, vectype, stmt_info, code, gsi, + slp_node, cost_vec)) return false; if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)) { if (direct_internal_fn_supported_p (IFN_VCOND_MASK_LEN, vectype, OPTIMIZE_FOR_SPEED)) - vect_record_loop_len (loop_vinfo, lens, ncopies * vec_num, - vectype, 1); + vect_record_loop_len (loop_vinfo, lens, vec_num, vectype, 1); else - vect_record_loop_mask (loop_vinfo, masks, ncopies * vec_num, - vectype, NULL); + vect_record_loop_mask (loop_vinfo, masks, vec_num, vectype, NULL); } return true; @@ -13690,28 +12914,13 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info, vectorized. It's not very clean to do this here, But the masking code below is complex and this keeps it all in one place to ease fixes and backports. Once we drop the non-SLP loop vect or split vectorizable_* this can be simplified. */ - if (!slp_node) - { - if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi, - vec_stmt, slp_node, cost_vec)) - gcc_unreachable (); - } gimple *stmt = STMT_VINFO_STMT (stmt_info); basic_block cond_bb = gimple_bb (stmt); gimple_stmt_iterator cond_gsi = gsi_last_bb (cond_bb); auto_vec<tree> stmts; - - if (slp_node) - stmts.safe_splice (SLP_TREE_VEC_DEFS (slp_node)); - else - { - auto vec_stmts = STMT_VINFO_VEC_STMTS (stmt_info); - stmts.reserve_exact (vec_stmts.length ()); - for (auto stmt : vec_stmts) - stmts.quick_push (gimple_assign_lhs (stmt)); - } + stmts.safe_splice (SLP_TREE_VEC_DEFS (slp_node)); /* If we're comparing against a previous forall we need to negate the resullts before we do the final comparison or reduction. */ @@ -13750,7 +12959,7 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info, for (unsigned i = 0; i < stmts.length (); i++) { tree stmt_mask - = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies * vec_num, + = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num, vectype, i); stmt_mask = prepare_vec_mask (loop_vinfo, TREE_TYPE (stmt_mask), stmt_mask, @@ -13761,7 +12970,7 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info, for (unsigned i = 0; i < stmts.length (); i++) { tree len_mask = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi, - lens, ncopies * vec_num, + lens, vec_num, vectype, stmts[i], i, 1); workset.quick_push (len_mask); @@ -13771,10 +12980,22 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info, while (workset.length () > 1) { - new_temp = make_temp_ssa_name (vectype, NULL, "vexit_reduc"); tree arg0 = workset.pop (); tree arg1 = workset.pop (); - new_stmt = gimple_build_assign (new_temp, BIT_IOR_EXPR, arg0, arg1); + if (addhn_supported_p && workset.length () == 0) + { + new_stmt = gimple_build_call_internal (ifn, 2, arg0, arg1); + vectype_out = narrow_type; + new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc"); + gimple_call_set_lhs (as_a <gcall *> (new_stmt), new_temp); + gimple_call_set_nothrow (as_a <gcall *> (new_stmt), true); + } + else + { + new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc"); + new_stmt + = gimple_build_assign (new_temp, BIT_IOR_EXPR, arg0, arg1); + } vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, &cond_gsi); workset.quick_insert (0, new_temp); @@ -13786,27 +13007,23 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info, if (masked_loop_p) { tree mask - = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies, vectype, 0); + = vect_get_loop_mask (loop_vinfo, gsi, masks, 1, vectype, 0); new_temp = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask, new_temp, &cond_gsi); } else if (len_loop_p) new_temp = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi, lens, - ncopies, vectype, new_temp, 0, 1); + 1, vectype, new_temp, 0, 1); } gcc_assert (new_temp); + tree cst = build_zero_cst (vectype_out); gimple_cond_set_condition (cond_stmt, NE_EXPR, new_temp, cst); update_stmt (orig_stmt); - if (slp_node) - SLP_TREE_VEC_DEFS (slp_node).truncate (0); - else - STMT_VINFO_VEC_STMTS (stmt_info).truncate (0); - - if (!slp_node) - *vec_stmt = orig_stmt; + /* ??? */ + SLP_TREE_VEC_DEFS (slp_node).truncate (0); return true; } @@ -13817,37 +13034,22 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info, VEC_STMT_P is as for vectorizable_live_operation. */ static bool -can_vectorize_live_stmts (vec_info *vinfo, stmt_vec_info stmt_info, +can_vectorize_live_stmts (vec_info *vinfo, slp_tree slp_node, slp_instance slp_node_instance, bool vec_stmt_p, stmt_vector_for_cost *cost_vec) { - loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); - if (slp_node) - { - stmt_vec_info slp_stmt_info; - unsigned int i; - FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info) - { - if (slp_stmt_info - && (STMT_VINFO_LIVE_P (slp_stmt_info) - || (loop_vinfo - && LOOP_VINFO_EARLY_BREAKS (loop_vinfo) - && STMT_VINFO_DEF_TYPE (slp_stmt_info) - == vect_induction_def)) - && !vectorizable_live_operation (vinfo, slp_stmt_info, slp_node, - slp_node_instance, i, - vec_stmt_p, cost_vec)) - return false; - } + stmt_vec_info slp_stmt_info; + unsigned int i; + FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info) + { + if (slp_stmt_info + && STMT_VINFO_LIVE_P (slp_stmt_info) + && !vectorizable_live_operation (vinfo, slp_stmt_info, slp_node, + slp_node_instance, i, + vec_stmt_p, cost_vec)) + return false; } - else if ((STMT_VINFO_LIVE_P (stmt_info) - || (LOOP_VINFO_EARLY_BREAKS (loop_vinfo) - && STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)) - && !vectorizable_live_operation (vinfo, stmt_info, - slp_node, slp_node_instance, -1, - vec_stmt_p, cost_vec)) - return false; return true; } @@ -13856,115 +13058,42 @@ can_vectorize_live_stmts (vec_info *vinfo, stmt_vec_info stmt_info, opt_result vect_analyze_stmt (vec_info *vinfo, - stmt_vec_info stmt_info, bool *need_to_vectorize, slp_tree node, slp_instance node_instance, stmt_vector_for_cost *cost_vec) { + stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node); bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo); enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info); bool ok; - gimple_seq pattern_def_seq; if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G", stmt_info->stmt); if (gimple_has_volatile_ops (stmt_info->stmt)) - return opt_result::failure_at (stmt_info->stmt, - "not vectorized:" - " stmt has volatile operands: %G\n", - stmt_info->stmt); - - if (STMT_VINFO_IN_PATTERN_P (stmt_info) - && node == NULL - && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info))) { - gimple_stmt_iterator si; - - for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si)) - { - stmt_vec_info pattern_def_stmt_info - = vinfo->lookup_stmt (gsi_stmt (si)); - if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info) - || STMT_VINFO_LIVE_P (pattern_def_stmt_info)) - { - /* Analyze def stmt of STMT if it's a pattern stmt. */ - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "==> examining pattern def statement: %G", - pattern_def_stmt_info->stmt); - - opt_result res - = vect_analyze_stmt (vinfo, pattern_def_stmt_info, - need_to_vectorize, node, node_instance, - cost_vec); - if (!res) - return res; - } - } + /* ??? This shouldn't really happen, volatile stmts should + not end up in the SLP graph. */ + return opt_result::failure_at (stmt_info->stmt, + "not vectorized:" + " stmt has volatile operands: %G\n", + stmt_info->stmt); } - /* Skip stmts that do not need to be vectorized. In loops this is expected - to include: - - the COND_EXPR which is the loop exit condition - - any LABEL_EXPRs in the loop - - computations that are used only for array indexing or loop control. - In basic blocks we only analyze statements that are a part of some SLP - instance, therefore, all the statements are relevant. - - Pattern statement needs to be analyzed instead of the original statement - if the original statement is not relevant. Otherwise, we analyze both - statements. In basic blocks we are called from some SLP instance - traversal, don't analyze pattern stmts instead, the pattern stmts - already will be part of SLP instance. */ - - stmt_vec_info pattern_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info); + /* Skip stmts that do not need to be vectorized. */ if (!STMT_VINFO_RELEVANT_P (stmt_info) && !STMT_VINFO_LIVE_P (stmt_info)) { - if (STMT_VINFO_IN_PATTERN_P (stmt_info) - && pattern_stmt_info - && (STMT_VINFO_RELEVANT_P (pattern_stmt_info) - || STMT_VINFO_LIVE_P (pattern_stmt_info))) - { - /* Analyze PATTERN_STMT instead of the original stmt. */ - stmt_info = pattern_stmt_info; - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "==> examining pattern statement: %G", - stmt_info->stmt); - } - else - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n"); - - if (node) - return opt_result::failure_at (stmt_info->stmt, - "not vectorized:" - " irrelevant stmt as SLP node %p " - "representative.\n", - (void *)node); - return opt_result::success (); - } - } - else if (STMT_VINFO_IN_PATTERN_P (stmt_info) - && node == NULL - && pattern_stmt_info - && (STMT_VINFO_RELEVANT_P (pattern_stmt_info) - || STMT_VINFO_LIVE_P (pattern_stmt_info))) - { - /* Analyze PATTERN_STMT too. */ if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "==> examining pattern statement: %G", - pattern_stmt_info->stmt); + dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n"); - opt_result res - = vect_analyze_stmt (vinfo, pattern_stmt_info, need_to_vectorize, node, - node_instance, cost_vec); - if (!res) - return res; + /* ??? This shouldn't really happen, irrelevant stmts should + not end up in the SLP graph. */ + return opt_result::failure_at (stmt_info->stmt, + "not vectorized:" + " irrelevant stmt as SLP node %p " + "representative.\n", + (void *)node); } switch (STMT_VINFO_DEF_TYPE (stmt_info)) @@ -14000,97 +13129,53 @@ vect_analyze_stmt (vec_info *vinfo, } tree saved_vectype = STMT_VINFO_VECTYPE (stmt_info); - if (node) - STMT_VINFO_VECTYPE (stmt_info) = SLP_TREE_VECTYPE (node); + STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE; if (STMT_VINFO_RELEVANT_P (stmt_info)) { gcall *call = dyn_cast <gcall *> (stmt_info->stmt); - gcc_assert (STMT_VINFO_VECTYPE (stmt_info) + gcc_assert (SLP_TREE_VECTYPE (node) || gimple_code (stmt_info->stmt) == GIMPLE_COND || (call && gimple_call_lhs (call) == NULL_TREE)); - *need_to_vectorize = true; - } - - if (PURE_SLP_STMT (stmt_info) && !node) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "handled only by SLP analysis\n"); - return opt_result::success (); } - /* When we arrive here with a non-SLP statement and we are supposed - to use SLP for everything fail vectorization. */ - if (!node) - return opt_result::failure_at (stmt_info->stmt, - "needs non-SLP handling\n"); - ok = true; - if (!bb_vinfo - && (STMT_VINFO_RELEVANT_P (stmt_info) + if (bb_vinfo + || (STMT_VINFO_RELEVANT_P (stmt_info) || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)) /* Prefer vectorizable_call over vectorizable_simd_clone_call so -mveclibabi= takes preference over library functions with the simd attribute. */ - ok = (vectorizable_call (vinfo, stmt_info, NULL, NULL, node, cost_vec) - || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, NULL, node, + ok = (vectorizable_call (vinfo, stmt_info, NULL, node, cost_vec) + || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, node, cost_vec) - || vectorizable_conversion (vinfo, stmt_info, - NULL, NULL, node, cost_vec) - || vectorizable_operation (vinfo, stmt_info, - NULL, NULL, node, cost_vec) - || vectorizable_assignment (vinfo, stmt_info, - NULL, NULL, node, cost_vec) - || vectorizable_load (vinfo, stmt_info, NULL, NULL, node, cost_vec) - || vectorizable_store (vinfo, stmt_info, NULL, NULL, node, cost_vec) - || vectorizable_lane_reducing (as_a <loop_vec_info> (vinfo), - stmt_info, node, cost_vec) - || vectorizable_reduction (as_a <loop_vec_info> (vinfo), stmt_info, - node, node_instance, cost_vec) - || vectorizable_induction (as_a <loop_vec_info> (vinfo), stmt_info, - NULL, node, cost_vec) - || vectorizable_shift (vinfo, stmt_info, NULL, NULL, node, cost_vec) - || vectorizable_condition (vinfo, stmt_info, - NULL, NULL, node, cost_vec) - || vectorizable_comparison (vinfo, stmt_info, NULL, NULL, node, - cost_vec) - || vectorizable_lc_phi (as_a <loop_vec_info> (vinfo), - stmt_info, NULL, node) - || vectorizable_recurr (as_a <loop_vec_info> (vinfo), - stmt_info, NULL, node, cost_vec) - || vectorizable_early_exit (vinfo, stmt_info, NULL, NULL, node, - cost_vec)); - else - { - if (bb_vinfo) - ok = (vectorizable_call (vinfo, stmt_info, NULL, NULL, node, cost_vec) - || vectorizable_simd_clone_call (vinfo, stmt_info, - NULL, NULL, node, cost_vec) - || vectorizable_conversion (vinfo, stmt_info, NULL, NULL, node, - cost_vec) - || vectorizable_shift (vinfo, stmt_info, - NULL, NULL, node, cost_vec) - || vectorizable_operation (vinfo, stmt_info, - NULL, NULL, node, cost_vec) - || vectorizable_assignment (vinfo, stmt_info, NULL, NULL, node, - cost_vec) - || vectorizable_load (vinfo, stmt_info, - NULL, NULL, node, cost_vec) - || vectorizable_store (vinfo, stmt_info, - NULL, NULL, node, cost_vec) - || vectorizable_condition (vinfo, stmt_info, - NULL, NULL, node, cost_vec) - || vectorizable_comparison (vinfo, stmt_info, NULL, NULL, node, - cost_vec) - || vectorizable_phi (vinfo, stmt_info, NULL, node, cost_vec) - || vectorizable_early_exit (vinfo, stmt_info, NULL, NULL, node, - cost_vec)); - - } - - if (node) - STMT_VINFO_VECTYPE (stmt_info) = saved_vectype; + || vectorizable_conversion (vinfo, stmt_info, NULL, node, cost_vec) + || vectorizable_operation (vinfo, stmt_info, NULL, node, cost_vec) + || vectorizable_assignment (vinfo, stmt_info, NULL, node, cost_vec) + || vectorizable_load (vinfo, stmt_info, NULL, node, cost_vec) + || vectorizable_store (vinfo, stmt_info, NULL, node, cost_vec) + || vectorizable_shift (vinfo, stmt_info, NULL, node, cost_vec) + || vectorizable_condition (vinfo, stmt_info, NULL, node, cost_vec) + || vectorizable_comparison (vinfo, stmt_info, NULL, node, cost_vec) + || (bb_vinfo + && vectorizable_phi (bb_vinfo, stmt_info, node, cost_vec)) + || (is_a <loop_vec_info> (vinfo) + && (vectorizable_lane_reducing (as_a <loop_vec_info> (vinfo), + stmt_info, node, cost_vec) + || vectorizable_reduction (as_a <loop_vec_info> (vinfo), + stmt_info, + node, node_instance, cost_vec) + || vectorizable_induction (as_a <loop_vec_info> (vinfo), + stmt_info, node, cost_vec) + || vectorizable_lc_phi (as_a <loop_vec_info> (vinfo), + stmt_info, node) + || vectorizable_recurr (as_a <loop_vec_info> (vinfo), + stmt_info, node, cost_vec) + || vectorizable_early_exit (as_a <loop_vec_info> (vinfo), + stmt_info, NULL, node, + cost_vec)))); + + STMT_VINFO_VECTYPE (stmt_info) = saved_vectype; if (!ok) return opt_result::failure_at (stmt_info->stmt, @@ -14101,11 +13186,12 @@ vect_analyze_stmt (vec_info *vinfo, /* Stmts that are (also) "live" (i.e. - that are used out of the loop) need extra handling, except for vectorizable reductions. */ if (!bb_vinfo - && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type - && STMT_VINFO_TYPE (stmt_info) != lc_phi_info_type - && (!node || !node->ldst_lanes || SLP_TREE_CODE (node) == VEC_PERM_EXPR) + && SLP_TREE_TYPE (node) != reduc_vec_info_type + && (SLP_TREE_TYPE (node) != lc_phi_info_type + || SLP_TREE_DEF_TYPE (node) == vect_internal_def) + && (!node->ldst_lanes || SLP_TREE_PERMUTE_P (node)) && !can_vectorize_live_stmts (as_a <loop_vec_info> (vinfo), - stmt_info, node, node_instance, + node, node_instance, false, cost_vec)) return opt_result::failure_at (stmt_info->stmt, "not vectorized:" @@ -14126,133 +13212,111 @@ vect_transform_stmt (vec_info *vinfo, slp_tree slp_node, slp_instance slp_node_instance) { bool is_store = false; - gimple *vec_stmt = NULL; bool done; - gcc_assert (slp_node || !PURE_SLP_STMT (stmt_info)); + gcc_assert (slp_node); - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "------>vectorizing statement: %G", stmt_info->stmt); - - tree saved_vectype = STMT_VINFO_VECTYPE (stmt_info); - if (slp_node) - STMT_VINFO_VECTYPE (stmt_info) = SLP_TREE_VECTYPE (slp_node); + if (stmt_info) + STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE; - switch (STMT_VINFO_TYPE (stmt_info)) + switch (SLP_TREE_TYPE (slp_node)) { case type_demotion_vec_info_type: case type_promotion_vec_info_type: case type_conversion_vec_info_type: - done = vectorizable_conversion (vinfo, stmt_info, - gsi, &vec_stmt, slp_node, NULL); + done = vectorizable_conversion (vinfo, stmt_info, gsi, slp_node, NULL); gcc_assert (done); break; case induc_vec_info_type: done = vectorizable_induction (as_a <loop_vec_info> (vinfo), - stmt_info, &vec_stmt, slp_node, - NULL); + stmt_info, slp_node, NULL); gcc_assert (done); break; case shift_vec_info_type: - done = vectorizable_shift (vinfo, stmt_info, - gsi, &vec_stmt, slp_node, NULL); + done = vectorizable_shift (vinfo, stmt_info, gsi, slp_node, NULL); gcc_assert (done); break; case op_vec_info_type: - done = vectorizable_operation (vinfo, stmt_info, gsi, &vec_stmt, slp_node, - NULL); + done = vectorizable_operation (vinfo, stmt_info, gsi, slp_node, NULL); gcc_assert (done); break; case assignment_vec_info_type: - done = vectorizable_assignment (vinfo, stmt_info, - gsi, &vec_stmt, slp_node, NULL); + done = vectorizable_assignment (vinfo, stmt_info, gsi, slp_node, NULL); gcc_assert (done); break; case load_vec_info_type: - done = vectorizable_load (vinfo, stmt_info, gsi, &vec_stmt, slp_node, - NULL); + done = vectorizable_load (vinfo, stmt_info, gsi, slp_node, NULL); gcc_assert (done); break; case store_vec_info_type: - if (STMT_VINFO_GROUPED_ACCESS (stmt_info) - && !slp_node - && (++DR_GROUP_STORE_COUNT (DR_GROUP_FIRST_ELEMENT (stmt_info)) - < DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info)))) - /* In case of interleaving, the whole chain is vectorized when the - last store in the chain is reached. Store stmts before the last - one are skipped, and there vec_stmt_info shouldn't be freed - meanwhile. */ - ; - else - { - done = vectorizable_store (vinfo, stmt_info, - gsi, &vec_stmt, slp_node, NULL); - gcc_assert (done); - is_store = true; - } + done = vectorizable_store (vinfo, stmt_info, gsi, slp_node, NULL); + gcc_assert (done); + is_store = true; break; case condition_vec_info_type: - done = vectorizable_condition (vinfo, stmt_info, - gsi, &vec_stmt, slp_node, NULL); + done = vectorizable_condition (vinfo, stmt_info, gsi, slp_node, NULL); gcc_assert (done); break; case comparison_vec_info_type: - done = vectorizable_comparison (vinfo, stmt_info, gsi, &vec_stmt, - slp_node, NULL); + done = vectorizable_comparison (vinfo, stmt_info, gsi, slp_node, NULL); gcc_assert (done); break; case call_vec_info_type: - done = vectorizable_call (vinfo, stmt_info, - gsi, &vec_stmt, slp_node, NULL); + done = vectorizable_call (vinfo, stmt_info, gsi, slp_node, NULL); break; case call_simd_clone_vec_info_type: - done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi, &vec_stmt, + done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi, slp_node, NULL); break; case reduc_vec_info_type: done = vect_transform_reduction (as_a <loop_vec_info> (vinfo), stmt_info, - gsi, &vec_stmt, slp_node); + gsi, slp_node); gcc_assert (done); break; case cycle_phi_info_type: done = vect_transform_cycle_phi (as_a <loop_vec_info> (vinfo), stmt_info, - &vec_stmt, slp_node, slp_node_instance); + slp_node, slp_node_instance); gcc_assert (done); break; case lc_phi_info_type: - done = vectorizable_lc_phi (as_a <loop_vec_info> (vinfo), - stmt_info, &vec_stmt, slp_node); + done = vect_transform_lc_phi (as_a <loop_vec_info> (vinfo), + stmt_info, slp_node); gcc_assert (done); break; case recurr_info_type: done = vectorizable_recurr (as_a <loop_vec_info> (vinfo), - stmt_info, &vec_stmt, slp_node, NULL); + stmt_info, slp_node, NULL); gcc_assert (done); break; case phi_info_type: - done = vectorizable_phi (vinfo, stmt_info, &vec_stmt, slp_node, NULL); + done = vectorizable_phi (as_a <bb_vec_info> (vinfo), + stmt_info, slp_node, NULL); gcc_assert (done); break; case loop_exit_ctrl_vec_info_type: - done = vectorizable_early_exit (vinfo, stmt_info, gsi, &vec_stmt, - slp_node, NULL); + done = vectorizable_early_exit (as_a <loop_vec_info> (vinfo), + stmt_info, gsi, slp_node, NULL); + gcc_assert (done); + break; + + case permute_info_type: + done = vectorizable_slp_permutation (vinfo, gsi, slp_node, NULL); gcc_assert (done); break; @@ -14267,24 +13331,16 @@ vect_transform_stmt (vec_info *vinfo, done = true; } - if (!slp_node && vec_stmt) - gcc_assert (STMT_VINFO_VEC_STMTS (stmt_info).exists ()); - - if (STMT_VINFO_TYPE (stmt_info) != store_vec_info_type - && (!slp_node - || !slp_node->ldst_lanes - || SLP_TREE_CODE (slp_node) == VEC_PERM_EXPR)) + if (SLP_TREE_TYPE (slp_node) != store_vec_info_type + && (!slp_node->ldst_lanes || SLP_TREE_PERMUTE_P (slp_node))) { /* Handle stmts whose DEF is used outside the loop-nest that is being vectorized. */ - done = can_vectorize_live_stmts (vinfo, stmt_info, slp_node, + done = can_vectorize_live_stmts (vinfo, slp_node, slp_node_instance, true, NULL); gcc_assert (done); } - if (slp_node) - STMT_VINFO_VECTYPE (stmt_info) = saved_vectype; - return is_store; } @@ -14580,6 +13636,23 @@ vect_chooses_same_modes_p (vec_info *vinfo, machine_mode vector_mode) return true; } +/* Return true if replacing VECTOR_MODE with ALT_VECTOR_MODE would not + change the chosen vector modes for analysis of a loop. */ + +bool +vect_chooses_same_modes_p (machine_mode vector_mode, + machine_mode alt_vector_mode) +{ + return (VECTOR_MODE_P (vector_mode) + && VECTOR_MODE_P (alt_vector_mode) + && (related_vector_mode (vector_mode, + GET_MODE_INNER (alt_vector_mode)) + == alt_vector_mode) + && (related_vector_mode (alt_vector_mode, + GET_MODE_INNER (vector_mode)) + == vector_mode)); +} + /* Function vect_is_simple_use. Input: @@ -14701,123 +13774,49 @@ vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt, /* Function vect_is_simple_use. - Same as vect_is_simple_use but also determines the vector operand - type of OPERAND and stores it to *VECTYPE. If the definition of - OPERAND is vect_uninitialized_def, vect_constant_def or - vect_external_def *VECTYPE will be set to NULL_TREE and the caller - is responsible to compute the best suited vector type for the - scalar operand. */ - -bool -vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt, - tree *vectype, stmt_vec_info *def_stmt_info_out, - gimple **def_stmt_out) -{ - stmt_vec_info def_stmt_info; - gimple *def_stmt; - if (!vect_is_simple_use (operand, vinfo, dt, &def_stmt_info, &def_stmt)) - return false; - - if (def_stmt_out) - *def_stmt_out = def_stmt; - if (def_stmt_info_out) - *def_stmt_info_out = def_stmt_info; - - /* Now get a vector type if the def is internal, otherwise supply - NULL_TREE and leave it up to the caller to figure out a proper - type for the use stmt. */ - if (*dt == vect_internal_def - || *dt == vect_induction_def - || *dt == vect_reduction_def - || *dt == vect_double_reduction_def - || *dt == vect_nested_cycle - || *dt == vect_first_order_recurrence) - { - *vectype = STMT_VINFO_VECTYPE (def_stmt_info); - gcc_assert (*vectype != NULL_TREE); - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "vect_is_simple_use: vectype %T\n", *vectype); - } - else if (*dt == vect_uninitialized_def - || *dt == vect_constant_def - || *dt == vect_external_def) - *vectype = NULL_TREE; - else - gcc_unreachable (); - - return true; -} - -/* Function vect_is_simple_use. - Same as vect_is_simple_use but determines the operand by operand position OPERAND from either STMT or SLP_NODE, filling in *OP and *SLP_DEF (when SLP_NODE is not NULL). */ bool -vect_is_simple_use (vec_info *vinfo, stmt_vec_info stmt, slp_tree slp_node, +vect_is_simple_use (vec_info *vinfo, slp_tree slp_node, unsigned operand, tree *op, slp_tree *slp_def, enum vect_def_type *dt, tree *vectype, stmt_vec_info *def_stmt_info_out) { - if (slp_node) - { - slp_tree child = SLP_TREE_CHILDREN (slp_node)[operand]; - *slp_def = child; - *vectype = SLP_TREE_VECTYPE (child); - if (SLP_TREE_DEF_TYPE (child) == vect_internal_def) - { - /* ??? VEC_PERM nodes might be intermediate and their lane value - have no representative (nor do we build a VEC_PERM stmt for - the actual operation). Note for two-operator nodes we set - a representative but leave scalar stmts empty as we'd only - have one for a subset of lanes. Ideally no caller would - require *op for internal defs. */ - if (SLP_TREE_REPRESENTATIVE (child)) - { - *op = gimple_get_lhs (SLP_TREE_REPRESENTATIVE (child)->stmt); - return vect_is_simple_use (*op, vinfo, dt, def_stmt_info_out); - } - else - { - gcc_assert (SLP_TREE_CODE (child) == VEC_PERM_EXPR); - *op = error_mark_node; - *dt = vect_internal_def; - if (def_stmt_info_out) - *def_stmt_info_out = NULL; - return true; - } + slp_tree child = SLP_TREE_CHILDREN (slp_node)[operand]; + *slp_def = child; + *vectype = SLP_TREE_VECTYPE (child); + if (SLP_TREE_DEF_TYPE (child) == vect_internal_def) + { + /* ??? VEC_PERM nodes might be intermediate and their lane value + have no representative (nor do we build a VEC_PERM stmt for + the actual operation). Note for two-operator nodes we set + a representative but leave scalar stmts empty as we'd only + have one for a subset of lanes. Ideally no caller would + require *op for internal defs. */ + if (SLP_TREE_REPRESENTATIVE (child)) + { + *op = gimple_get_lhs (SLP_TREE_REPRESENTATIVE (child)->stmt); + return vect_is_simple_use (*op, vinfo, dt, def_stmt_info_out); } else { + gcc_assert (SLP_TREE_PERMUTE_P (child)); + *op = error_mark_node; + *dt = vect_internal_def; if (def_stmt_info_out) *def_stmt_info_out = NULL; - *op = SLP_TREE_SCALAR_OPS (child)[0]; - *dt = SLP_TREE_DEF_TYPE (child); return true; } } else { - *slp_def = NULL; - if (gassign *ass = dyn_cast <gassign *> (stmt->stmt)) - { - if (gimple_assign_rhs_code (ass) == COND_EXPR - && COMPARISON_CLASS_P (gimple_assign_rhs1 (ass))) - gcc_unreachable (); - else if (gimple_assign_rhs_code (ass) == VIEW_CONVERT_EXPR) - *op = TREE_OPERAND (gimple_assign_rhs1 (ass), 0); - else - *op = gimple_op (ass, operand + 1); - } - else if (gcond *cond = dyn_cast <gcond *> (stmt->stmt)) - *op = gimple_op (cond, operand); - else if (gcall *call = dyn_cast <gcall *> (stmt->stmt)) - *op = gimple_call_arg (call, operand); - else - gcc_unreachable (); - return vect_is_simple_use (*op, vinfo, dt, vectype, def_stmt_info_out); + if (def_stmt_info_out) + *def_stmt_info_out = NULL; + *op = SLP_TREE_SCALAR_OPS (child)[0]; + *dt = SLP_TREE_DEF_TYPE (child); + return true; } } @@ -14856,6 +13855,8 @@ vect_maybe_update_slp_op_vectype (slp_tree op, tree vectype) are supported by the target platform either directly (via vector tree-codes), or via target builtins. + When EVENODD_OK then also lane-swizzling operations are considered. + Output: - CODE1 and CODE2 are codes of vector operations to be used when vectorizing the operation, if available. @@ -14866,17 +13867,14 @@ vect_maybe_update_slp_op_vectype (slp_tree op, tree vectype) widening operation (short in the above example). */ bool -supportable_widening_operation (vec_info *vinfo, - code_helper code, - stmt_vec_info stmt_info, +supportable_widening_operation (code_helper code, tree vectype_out, tree vectype_in, + bool evenodd_ok, code_helper *code1, code_helper *code2, int *multi_step_cvt, vec<tree> *interm_types) { - loop_vec_info loop_info = dyn_cast <loop_vec_info> (vinfo); - class loop *vect_loop = NULL; machine_mode vec_mode; enum insn_code icode1, icode2; optab optab1 = unknown_optab, optab2 = unknown_optab; @@ -14889,8 +13887,6 @@ supportable_widening_operation (vec_info *vinfo, optab optab3, optab4; *multi_step_cvt = 0; - if (loop_info) - vect_loop = LOOP_VINFO_LOOP (loop_info); switch (code.safe_as_tree_code ()) { @@ -14932,24 +13928,13 @@ supportable_widening_operation (vec_info *vinfo, on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values are properly set up for the caller. If we fail, we'll continue with a VEC_WIDEN_MULT_LO/HI_EXPR check. */ - if (vect_loop - && !nested_in_vect_loop_p (vect_loop, stmt_info) - && supportable_widening_operation (vinfo, VEC_WIDEN_MULT_EVEN_EXPR, - stmt_info, vectype_out, - vectype_in, code1, + if (evenodd_ok + && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR, + vectype_out, vectype_in, + evenodd_ok, code1, code2, multi_step_cvt, interm_types)) - { - /* Elements in a vector with vect_used_by_reduction property cannot - be reordered if the use chain with this property does not have the - same operation. One such an example is s += a * b, where elements - in a and b cannot be reordered. Here we check if the vector defined - by STMT is only directly used in the reduction statement. */ - tree lhs = gimple_assign_lhs (vect_orig_stmt (stmt_info)->stmt); - stmt_vec_info use_stmt_info = loop_info->lookup_single_use (lhs); - if (use_stmt_info && STMT_VINFO_REDUC_DEF (use_stmt_info)) - return true; - } + return true; c1 = VEC_WIDEN_MULT_LO_EXPR; c2 = VEC_WIDEN_MULT_HI_EXPR; break; @@ -15024,6 +14009,8 @@ supportable_widening_operation (vec_info *vinfo, internal_fn lo, hi, even, odd; lookup_hilo_internal_fn (ifn, &lo, &hi); + if (BYTES_BIG_ENDIAN) + std::swap (lo, hi); *code1 = as_combined_fn (lo); *code2 = as_combined_fn (hi); optab1 = direct_internal_fn_optab (lo, {vectype, vectype}); @@ -15126,9 +14113,9 @@ supportable_widening_operation (vec_info *vinfo, TYPE_UNSIGNED (prev_type)); if (VECTOR_BOOLEAN_TYPE_P (intermediate_type) - && VECTOR_BOOLEAN_TYPE_P (prev_type) - && intermediate_mode == prev_mode - && SCALAR_INT_MODE_P (prev_mode)) + && VECTOR_BOOLEAN_TYPE_P (wide_vectype) + && intermediate_mode == TYPE_MODE (wide_vectype) + && SCALAR_INT_MODE_P (intermediate_mode)) { /* If the input and result modes are the same, a different optab is needed where we pass in the number of units in vectype. */ @@ -15377,7 +14364,6 @@ supportable_indirect_convert_operation (code_helper code, bool found_mode = false; scalar_mode lhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_out)); scalar_mode rhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_in)); - opt_scalar_mode mode_iter; tree_code tc1, tc2, code1, code2; tree cvt_type = NULL_TREE; |