diff options
author | Richard Biener <rguenther@suse.de> | 2023-07-24 12:14:26 +0200 |
---|---|---|
committer | Richard Biener <rguenther@suse.de> | 2023-07-24 15:32:00 +0200 |
commit | 2b074dc7bdf4402c1e3e44f247e27f3ef51bd8e4 (patch) | |
tree | e3314c54ccfb9c37c61b0d4eedca1daea5b5e620 /gcc | |
parent | 8547c451aac1c1f0178cf0914243b20fa0dde0c3 (diff) | |
download | gcc-2b074dc7bdf4402c1e3e44f247e27f3ef51bd8e4.zip gcc-2b074dc7bdf4402c1e3e44f247e27f3ef51bd8e4.tar.gz gcc-2b074dc7bdf4402c1e3e44f247e27f3ef51bd8e4.tar.bz2 |
Remove SLP_TREE_VEC_STMTS in favor of SLP_TREE_VEC_DEFS
The following unifies SLP_TREE_VEC_STMTS into SLP_TREE_VEC_DEFS
which can handle all cases we need.
* tree-vectorizer.h (_slp_tree::push_vec_def): Add.
(_slp_tree::vec_stmts): Remove.
(SLP_TREE_VEC_STMTS): Remove.
* tree-vect-slp.cc (_slp_tree::push_vec_def): Define.
(_slp_tree::_slp_tree): Adjust.
(_slp_tree::~_slp_tree): Likewise.
(vect_get_slp_vect_def): Simplify.
(vect_get_slp_defs): Likewise.
(vect_transform_slp_perm_load_1): Adjust.
(vect_add_slp_permutation): Likewise.
(vect_schedule_slp_node): Likewise.
(vectorize_slp_instance_root_stmt): Likewise.
(vect_schedule_scc): Likewise.
* tree-vect-stmts.cc (vectorizable_bswap): Use push_vec_def.
(vectorizable_call): Likewise.
(vectorizable_call): Likewise.
(vect_create_vectorized_demotion_stmts): Likewise.
(vectorizable_conversion): Likewise.
(vectorizable_assignment): Likewise.
(vectorizable_shift): Likewise.
(vectorizable_operation): Likewise.
(vectorizable_load): Likewise.
(vectorizable_condition): Likewise.
(vectorizable_comparison): Likewise.
* tree-vect-loop.cc (vect_create_epilog_for_reduction): Adjust.
(vectorize_fold_left_reduction): Use push_vec_def.
(vect_transform_reduction): Likewise.
(vect_transform_cycle_phi): Likewise.
(vectorizable_lc_phi): Likewise.
(vectorizable_phi): Likewise.
(vectorizable_recurr): Likewise.
(vectorizable_induction): Likewise.
(vectorizable_live_operation): Likewise.
Diffstat (limited to 'gcc')
-rw-r--r-- | gcc/tree-vect-loop.cc | 31 | ||||
-rw-r--r-- | gcc/tree-vect-slp.cc | 91 | ||||
-rw-r--r-- | gcc/tree-vect-stmts.cc | 32 | ||||
-rw-r--r-- | gcc/tree-vectorizer.h | 7 |
4 files changed, 84 insertions, 77 deletions
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index ac86b09..2561552 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -5847,7 +5847,7 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, int ncopies; if (slp_node) { - vec_num = SLP_TREE_VEC_STMTS (slp_node_instance->reduc_phis).length (); + vec_num = SLP_TREE_VEC_DEFS (slp_node_instance->reduc_phis).length (); ncopies = 1; } else @@ -6990,7 +6990,7 @@ vectorize_fold_left_reduction (loop_vec_info loop_vinfo, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else { STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); @@ -8292,7 +8292,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo, } if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else if (single_defuse_cycle && i < ncopies - 1) { @@ -8578,7 +8578,7 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo, /* The loop-latch arg is set in epilogue processing. */ if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi); + slp_node->push_vec_def (new_phi); else { if (j == 0) @@ -8639,7 +8639,7 @@ vectorizable_lc_phi (loop_vec_info loop_vinfo, gphi *new_phi = create_phi_node (vec_dest, bb); add_phi_arg (new_phi, vec_oprnds[i], e, UNKNOWN_LOCATION); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi); + slp_node->push_vec_def (new_phi); else STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi); } @@ -8719,7 +8719,7 @@ vectorizable_phi (vec_info *, /* Skip not yet vectorized defs. */ if (SLP_TREE_DEF_TYPE (child) == vect_internal_def - && SLP_TREE_VEC_STMTS (child).is_empty ()) + && SLP_TREE_VEC_DEFS (child).is_empty ()) continue; auto_vec<tree> vec_oprnds; @@ -8731,7 +8731,7 @@ vectorizable_phi (vec_info *, { /* Create the vectorized LC PHI node. */ new_phis.quick_push (create_phi_node (vec_dest, bb)); - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phis[j]); + slp_node->push_vec_def (new_phis[j]); } } edge e = gimple_phi_arg_edge (as_a <gphi *> (stmt_info->stmt), i); @@ -8912,7 +8912,7 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info, vect_finish_stmt_generation (loop_vinfo, stmt_info, vperm, &gsi2); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (vperm); + slp_node->push_vec_def (vperm); else STMT_VINFO_VEC_STMTS (stmt_info).safe_push (vperm); } @@ -9868,7 +9868,7 @@ vectorizable_induction (loop_vec_info loop_vinfo, /* Set the arguments of the phi node: */ add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION); - SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi); + slp_node->push_vec_def (induction_phi); } if (!nested_in_vect_loop) { @@ -9878,8 +9878,7 @@ vectorizable_induction (loop_vec_info loop_vinfo, vec_steps.reserve (nivs-ivn); for (; ivn < nivs; ++ivn) { - SLP_TREE_VEC_STMTS (slp_node) - .quick_push (SLP_TREE_VEC_STMTS (slp_node)[0]); + slp_node->push_vec_def (SLP_TREE_VEC_DEFS (slp_node)[0]); vec_steps.quick_push (vec_steps[0]); } } @@ -9898,7 +9897,8 @@ vectorizable_induction (loop_vec_info loop_vinfo, : build_int_cstu (stept, vfp)); for (; ivn < nvects; ++ivn) { - gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]; + gimple *iv + = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (slp_node)[ivn - nivs]); tree def = gimple_get_lhs (iv); if (ivn < 2*nivs) vec_steps[ivn - nivs] @@ -9916,8 +9916,7 @@ vectorizable_induction (loop_vec_info loop_vinfo, gimple_stmt_iterator tgsi = gsi_for_stmt (iv); gsi_insert_seq_after (&tgsi, stmts, GSI_CONTINUE_LINKING); } - SLP_TREE_VEC_STMTS (slp_node) - .quick_push (SSA_NAME_DEF_STMT (def)); + slp_node->push_vec_def (def); } } @@ -10343,8 +10342,8 @@ vectorizable_live_operation (vec_info *vinfo, gcc_assert (!loop_vinfo || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)); /* Get the correct slp vectorized stmt. */ - vec_stmt = SLP_TREE_VEC_STMTS (slp_node)[vec_entry]; - vec_lhs = gimple_get_lhs (vec_stmt); + vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry]; + vec_stmt = SSA_NAME_DEF_STMT (vec_lhs); /* Get entry to use. */ bitstart = bitsize_int (vec_index); diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc index f4adbb2..e443024 100644 --- a/gcc/tree-vect-slp.cc +++ b/gcc/tree-vect-slp.cc @@ -112,7 +112,6 @@ _slp_tree::_slp_tree () slp_first_node = this; SLP_TREE_SCALAR_STMTS (this) = vNULL; SLP_TREE_SCALAR_OPS (this) = vNULL; - SLP_TREE_VEC_STMTS (this) = vNULL; SLP_TREE_VEC_DEFS (this) = vNULL; SLP_TREE_NUMBER_OF_VEC_STMTS (this) = 0; SLP_TREE_CHILDREN (this) = vNULL; @@ -141,7 +140,6 @@ _slp_tree::~_slp_tree () SLP_TREE_CHILDREN (this).release (); SLP_TREE_SCALAR_STMTS (this).release (); SLP_TREE_SCALAR_OPS (this).release (); - SLP_TREE_VEC_STMTS (this).release (); SLP_TREE_VEC_DEFS (this).release (); SLP_TREE_LOAD_PERMUTATION (this).release (); SLP_TREE_LANE_PERMUTATION (this).release (); @@ -149,6 +147,20 @@ _slp_tree::~_slp_tree () free (failed); } +/* Push the single SSA definition in DEF to the vector of vector defs. */ + +void +_slp_tree::push_vec_def (gimple *def) +{ + if (gphi *phi = dyn_cast <gphi *> (def)) + vec_defs.quick_push (gimple_phi_result (phi)); + else + { + def_operand_p defop = single_ssa_def_operand (def, SSA_OP_ALL_DEFS); + vec_defs.quick_push (get_def_from_ptr (defop)); + } +} + /* Recursively free the memory allocated for the SLP tree rooted at NODE. */ void @@ -8092,10 +8104,7 @@ vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node) tree vect_get_slp_vect_def (slp_tree slp_node, unsigned i) { - if (SLP_TREE_VEC_STMTS (slp_node).exists ()) - return gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[i]); - else - return SLP_TREE_VEC_DEFS (slp_node)[i]; + return SLP_TREE_VEC_DEFS (slp_node)[i]; } /* Get the vectorized definitions of SLP_NODE in *VEC_DEFS. */ @@ -8104,15 +8113,7 @@ void vect_get_slp_defs (slp_tree slp_node, vec<tree> *vec_defs) { vec_defs->create (SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node)); - if (SLP_TREE_DEF_TYPE (slp_node) == vect_internal_def) - { - unsigned j; - gimple *vec_def_stmt; - FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), j, vec_def_stmt) - vec_defs->quick_push (gimple_get_lhs (vec_def_stmt)); - } - else - vec_defs->splice (SLP_TREE_VEC_DEFS (slp_node)); + vec_defs->splice (SLP_TREE_VEC_DEFS (slp_node)); } /* Get N vectorized definitions for SLP_NODE. */ @@ -8170,8 +8171,8 @@ vect_transform_slp_perm_load_1 (vec_info *vinfo, slp_tree node, /* Initialize the vect stmts of NODE to properly insert the generated stmts later. */ if (! analyze_only) - for (unsigned i = SLP_TREE_VEC_STMTS (node).length (); i < nstmts; i++) - SLP_TREE_VEC_STMTS (node).quick_push (NULL); + for (unsigned i = SLP_TREE_VEC_DEFS (node).length (); i < nstmts; i++) + SLP_TREE_VEC_DEFS (node).quick_push (NULL_TREE); /* Generate permutation masks for every NODE. Number of masks for each NODE is equal to GROUP_SIZE. @@ -8337,7 +8338,7 @@ vect_transform_slp_perm_load_1 (vec_info *vinfo, slp_tree node, } /* Store the vector statement in NODE. */ - SLP_TREE_VEC_STMTS (node)[vect_stmts_counter++] = perm_stmt; + SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = perm_dest; } } else if (!analyze_only) @@ -8347,12 +8348,11 @@ vect_transform_slp_perm_load_1 (vec_info *vinfo, slp_tree node, tree first_vec = dr_chain[first_vec_index + ri]; /* If mask was NULL_TREE generate the requested identity transform. */ - gimple *perm_stmt = SSA_NAME_DEF_STMT (first_vec); if (dce_chain) bitmap_set_bit (used_defs, first_vec_index + ri); /* Store the vector statement in NODE. */ - SLP_TREE_VEC_STMTS (node)[vect_stmts_counter++] = perm_stmt; + SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = first_vec; } } @@ -8503,7 +8503,7 @@ vect_add_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi, } vect_finish_stmt_generation (vinfo, NULL, perm_stmt, gsi); /* Store the vector statement in NODE. */ - SLP_TREE_VEC_STMTS (node).quick_push (perm_stmt); + node->push_vec_def (perm_stmt); } /* Subroutine of vectorizable_slp_permutation. Check whether the target @@ -8822,10 +8822,11 @@ vect_schedule_slp_node (vec_info *vinfo, slp_tree child; /* For existing vectors there's nothing to do. */ - if (SLP_TREE_VEC_DEFS (node).exists ()) + if (SLP_TREE_DEF_TYPE (node) == vect_external_def + && SLP_TREE_VEC_DEFS (node).exists ()) return; - gcc_assert (SLP_TREE_VEC_STMTS (node).is_empty ()); + gcc_assert (SLP_TREE_VEC_DEFS (node).is_empty ()); /* Vectorize externals and constants. */ if (SLP_TREE_DEF_TYPE (node) == vect_constant_def @@ -8844,7 +8845,7 @@ vect_schedule_slp_node (vec_info *vinfo, stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node); gcc_assert (SLP_TREE_NUMBER_OF_VEC_STMTS (node) != 0); - SLP_TREE_VEC_STMTS (node).create (SLP_TREE_NUMBER_OF_VEC_STMTS (node)); + SLP_TREE_VEC_DEFS (node).create (SLP_TREE_NUMBER_OF_VEC_STMTS (node)); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -8885,7 +8886,7 @@ vect_schedule_slp_node (vec_info *vinfo, reduction PHI but we still have SLP_TREE_NUM_VEC_STMTS set so the representation isn't perfect. Resort to the last scalar def here. */ - if (SLP_TREE_VEC_STMTS (child).is_empty ()) + if (SLP_TREE_VEC_DEFS (child).is_empty ()) { gcc_assert (STMT_VINFO_TYPE (SLP_TREE_REPRESENTATIVE (child)) == cycle_phi_info_type); @@ -8900,11 +8901,14 @@ vect_schedule_slp_node (vec_info *vinfo, ??? Unless we have a load permutation applied and that figures to re-use an earlier generated load. */ unsigned j; - gimple *vstmt; - FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (child), j, vstmt) - if (!last_stmt - || vect_stmt_dominates_stmt_p (last_stmt, vstmt)) - last_stmt = vstmt; + tree vdef; + FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef) + { + gimple *vstmt = SSA_NAME_DEF_STMT (vdef); + if (!last_stmt + || vect_stmt_dominates_stmt_p (last_stmt, vstmt)) + last_stmt = vstmt; + } } else if (!SLP_TREE_VECTYPE (child)) { @@ -9069,8 +9073,7 @@ vectorize_slp_instance_root_stmt (slp_tree node, slp_instance instance) { if (SLP_TREE_NUMBER_OF_VEC_STMTS (node) == 1) { - gimple *child_stmt = SLP_TREE_VEC_STMTS (node)[0]; - tree vect_lhs = gimple_get_lhs (child_stmt); + tree vect_lhs = SLP_TREE_VEC_DEFS (node)[0]; tree root_lhs = gimple_get_lhs (instance->root_stmts[0]->stmt); if (!useless_type_conversion_p (TREE_TYPE (root_lhs), TREE_TYPE (vect_lhs))) @@ -9081,7 +9084,7 @@ vectorize_slp_instance_root_stmt (slp_tree node, slp_instance instance) else if (SLP_TREE_NUMBER_OF_VEC_STMTS (node) > 1) { int nelts = SLP_TREE_NUMBER_OF_VEC_STMTS (node); - gimple *child_stmt; + tree child_def; int j; vec<constructor_elt, va_gc> *v; vec_alloc (v, nelts); @@ -9089,9 +9092,8 @@ vectorize_slp_instance_root_stmt (slp_tree node, slp_instance instance) /* A CTOR can handle V16HI composition from VNx8HI so we do not need to convert vector elements if the types do not match. */ - FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt) - CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, - gimple_get_lhs (child_stmt)); + FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (node), j, child_def) + CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, child_def); tree lhs = gimple_get_lhs (instance->root_stmts[0]->stmt); tree rtype = TREE_TYPE (gimple_assign_rhs1 (instance->root_stmts[0]->stmt)); @@ -9282,26 +9284,31 @@ vect_schedule_scc (vec_info *vinfo, slp_tree node, slp_instance instance, child = SLP_TREE_CHILDREN (phi_node)[dest_idx]; if (!child || SLP_TREE_DEF_TYPE (child) != vect_internal_def) continue; - unsigned n = SLP_TREE_VEC_STMTS (phi_node).length (); + unsigned n = SLP_TREE_VEC_DEFS (phi_node).length (); /* Simply fill all args. */ if (STMT_VINFO_DEF_TYPE (SLP_TREE_REPRESENTATIVE (phi_node)) != vect_first_order_recurrence) for (unsigned i = 0; i < n; ++i) - add_phi_arg (as_a <gphi *> (SLP_TREE_VEC_STMTS (phi_node)[i]), - vect_get_slp_vect_def (child, i), - e, gimple_phi_arg_location (phi, dest_idx)); + { + tree phidef = SLP_TREE_VEC_DEFS (phi_node)[i]; + gphi *phi = as_a <gphi *> (SSA_NAME_DEF_STMT (phidef)); + add_phi_arg (phi, vect_get_slp_vect_def (child, i), + e, gimple_phi_arg_location (phi, dest_idx)); + } else { /* Unless it is a first order recurrence which needs args filled in for both the PHI node and the permutes. */ - gimple *perm = SLP_TREE_VEC_STMTS (phi_node)[0]; + gimple *perm + = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[0]); gimple *rphi = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (perm)); add_phi_arg (as_a <gphi *> (rphi), vect_get_slp_vect_def (child, n - 1), e, gimple_phi_arg_location (phi, dest_idx)); for (unsigned i = 0; i < n; ++i) { - gimple *perm = SLP_TREE_VEC_STMTS (phi_node)[i]; + gimple *perm + = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[i]); if (i > 0) gimple_assign_set_rhs1 (perm, vect_get_slp_vect_def (child, i - 1)); diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc index 15e12f2..99e61f8 100644 --- a/gcc/tree-vect-stmts.cc +++ b/gcc/tree-vect-stmts.cc @@ -3237,7 +3237,7 @@ vectorizable_bswap (vec_info *vinfo, vectype, tem2)); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } @@ -3694,7 +3694,7 @@ vectorizable_call (vec_info *vinfo, vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); new_stmt = call; } - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); } continue; } @@ -3823,7 +3823,7 @@ vectorizable_call (vec_info *vinfo, gimple_call_set_lhs (call, new_temp); gimple_call_set_nothrow (call, true); vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); - SLP_TREE_VEC_STMTS (slp_node).quick_push (call); + slp_node->push_vec_def (call); } continue; } @@ -4827,7 +4827,7 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds, vectors in SLP_NODE or in vector info of the scalar statement (or in STMT_VINFO_RELATED_STMT chain). */ if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } @@ -5564,7 +5564,7 @@ vectorizable_conversion (vec_info *vinfo, vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } @@ -5620,7 +5620,7 @@ vectorizable_conversion (vec_info *vinfo, new_stmt = SSA_NAME_DEF_STMT (vop0); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } @@ -5666,7 +5666,7 @@ vectorizable_conversion (vec_info *vinfo, vectors in SLP_NODE or in vector info of the scalar statement (or in STMT_VINFO_RELATED_STMT chain). */ if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } @@ -5865,7 +5865,7 @@ vectorizable_assignment (vec_info *vinfo, gimple_assign_set_lhs (new_stmt, new_temp); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } @@ -6323,7 +6323,7 @@ vectorizable_shift (vec_info *vinfo, gimple_assign_set_lhs (new_stmt, new_temp); vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } @@ -6985,7 +6985,7 @@ vectorizable_operation (vec_info *vinfo, } if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } @@ -9708,7 +9708,7 @@ vectorizable_load (vec_info *vinfo, gimple *new_stmt = SSA_NAME_DEF_STMT (new_temp); if (slp) for (j = 0; j < (int) SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); ++j) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else { for (j = 0; j < ncopies; ++j) @@ -9947,7 +9947,7 @@ vectorizable_load (vec_info *vinfo, if (slp_perm) dr_chain.quick_push (gimple_assign_lhs (new_stmt)); else - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); } else { @@ -10011,7 +10011,7 @@ vectorizable_load (vec_info *vinfo, /* Check if the chain of loads is already vectorized. */ if (STMT_VINFO_VEC_STMTS (first_stmt_info).exists () - /* For SLP we would need to copy over SLP_TREE_VEC_STMTS. + /* For SLP we would need to copy over SLP_TREE_VEC_DEFS. ??? But we can only do so if there is exactly one as we have no way to get at the rest. Leave the CSE opportunity alone. @@ -10997,7 +10997,7 @@ vectorizable_load (vec_info *vinfo, /* Store vector loads in the corresponding SLP_NODE. */ if (!costing_p && slp && !slp_perm) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); /* With SLP permutation we load the gaps as well, without we need to skip the gaps after we manage to fully load @@ -11705,7 +11705,7 @@ vectorizable_condition (vec_info *vinfo, vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } @@ -11942,7 +11942,7 @@ vectorizable_comparison (vec_info *vinfo, } } if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + slp_node->push_vec_def (new_stmt); else STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 6b1cf6d..a651614 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -174,6 +174,9 @@ struct _slp_tree { _slp_tree (); ~_slp_tree (); + void push_vec_def (gimple *def); + void push_vec_def (tree def) { vec_defs.quick_push (def); } + /* Nodes that contain def-stmts of this node statements operands. */ vec<slp_tree> children; @@ -194,8 +197,7 @@ struct _slp_tree { lane_permutation_t lane_permutation; tree vectype; - /* Vectorized stmt/s. */ - vec<gimple *> vec_stmts; + /* Vectorized defs. */ vec<tree> vec_defs; /* Number of vector stmts that are created to replace the group of scalar stmts. It is calculated during the transformation phase as the number of @@ -289,7 +291,6 @@ public: #define SLP_TREE_SCALAR_STMTS(S) (S)->stmts #define SLP_TREE_SCALAR_OPS(S) (S)->ops #define SLP_TREE_REF_COUNT(S) (S)->refcnt -#define SLP_TREE_VEC_STMTS(S) (S)->vec_stmts #define SLP_TREE_VEC_DEFS(S) (S)->vec_defs #define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size #define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation |