diff options
Diffstat (limited to 'gcc')
| -rw-r--r-- | gcc/tree-vect-data-refs.c | 8 | ||||
| -rw-r--r-- | gcc/tree-vect-loop.c | 112 | ||||
| -rw-r--r-- | gcc/tree-vect-patterns.c | 3 | ||||
| -rw-r--r-- | gcc/tree-vect-slp.c | 47 | ||||
| -rw-r--r-- | gcc/tree-vect-stmts.c | 461 | ||||
| -rw-r--r-- | gcc/tree-vectorizer.c | 5 | ||||
| -rw-r--r-- | gcc/tree-vectorizer.h | 26 |
7 files changed, 282 insertions, 380 deletions
diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 7edd9eb..39d5a1b 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -4896,7 +4896,6 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info, aggr_ptr, loop, &incr_gsi, insert_after, &indx_before_incr, &indx_after_incr); incr = gsi_stmt (incr_gsi); - loop_vinfo->add_stmt (incr); /* Copy the points-to information if it exists. */ if (DR_PTR_INFO (dr)) @@ -4926,7 +4925,6 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info, containing_loop, &incr_gsi, insert_after, &indx_before_incr, &indx_after_incr); incr = gsi_stmt (incr_gsi); - loop_vinfo->add_stmt (incr); /* Copy the points-to information if it exists. */ if (DR_PTR_INFO (dr)) @@ -6407,7 +6405,7 @@ vect_transform_grouped_load (vec_info *vinfo, stmt_vec_info stmt_info, for each vector to the associated scalar statement. */ void -vect_record_grouped_load_vectors (vec_info *vinfo, stmt_vec_info stmt_info, +vect_record_grouped_load_vectors (vec_info *, stmt_vec_info stmt_info, vec<tree> result_chain) { stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info); @@ -6441,10 +6439,10 @@ vect_record_grouped_load_vectors (vec_info *vinfo, stmt_vec_info stmt_info, DR_GROUP_SAME_DR_STMT. */ if (next_stmt_info) { - stmt_vec_info new_stmt_info = vinfo->lookup_def (tmp_data_ref); + gimple *new_stmt = SSA_NAME_DEF_STMT (tmp_data_ref); /* We assume that if VEC_STMT is not NULL, this is a case of multiple copies, and we put the new vector statement last. */ - STMT_VINFO_VEC_STMTS (next_stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (next_stmt_info).safe_push (new_stmt); next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info); gap_count = 1; diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index cc3d391..f0b3325 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -4585,7 +4585,7 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, { if (gimple_assign_rhs_code (cond_info->stmt) == COND_EXPR) { - gimple *vec_stmt = STMT_VINFO_VEC_STMTS (cond_info)[0]->stmt; + gimple *vec_stmt = STMT_VINFO_VEC_STMTS (cond_info)[0]; gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR); ccompares.safe_push (std::make_pair (unshare_expr (gimple_assign_rhs1 (vec_stmt)), @@ -4636,7 +4636,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, /* Create a vector phi node. */ tree new_phi_tree = make_ssa_name (cr_index_vector_type); new_phi = create_phi_node (new_phi_tree, loop->header); - loop_vinfo->add_stmt (new_phi); add_phi_arg (as_a <gphi *> (new_phi), vec_zero, loop_preheader_edge (loop), UNKNOWN_LOCATION); @@ -4663,9 +4662,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, new_phi_tree, indx_before_incr); } gsi_insert_seq_before (&incr_gsi, stmts, GSI_SAME_STMT); - stmt_vec_info index_vec_info - = loop_vinfo->add_stmt (SSA_NAME_DEF_STMT (new_phi_tree)); - STMT_VINFO_VECTYPE (index_vec_info) = cr_index_vector_type; /* Update the phi with the vec cond. */ induction_index = new_phi_tree; @@ -4712,17 +4708,16 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, if (slp_node) def = vect_get_slp_vect_def (slp_node, i); else - def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[0]->stmt); + def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[0]); for (j = 0; j < ncopies; j++) { tree new_def = copy_ssa_name (def); phi = create_phi_node (new_def, exit_bb); - loop_vinfo->add_stmt (phi); if (j == 0) new_phis.quick_push (phi); else { - def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[j]->stmt); + def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[j]); new_phis.quick_push (phi); } @@ -5446,10 +5441,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT); if (nested_in_vect_loop) { - stmt_vec_info epilog_stmt_info = loop_vinfo->add_stmt (epilog_stmt); - STMT_VINFO_RELATED_STMT (epilog_stmt_info) - = STMT_VINFO_RELATED_STMT (loop_vinfo->lookup_stmt (new_phi)); - if (!double_reduc) scalar_results.quick_push (new_temp); else @@ -5666,7 +5657,7 @@ static bool vectorize_fold_left_reduction (loop_vec_info loop_vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, gimple *reduc_def_stmt, tree_code code, internal_fn reduc_fn, tree ops[3], tree vectype_in, @@ -5674,7 +5665,6 @@ vectorize_fold_left_reduction (loop_vec_info loop_vinfo, { class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); tree vectype_out = STMT_VINFO_VECTYPE (stmt_info); - stmt_vec_info new_stmt_info = NULL; internal_fn mask_reduc_fn = get_masked_reduction_fn (reduc_fn, vectype_in); int ncopies; @@ -5785,23 +5775,22 @@ vectorize_fold_left_reduction (loop_vec_info loop_vinfo, if (i == vec_num - 1) { gimple_set_lhs (new_stmt, scalar_dest); - new_stmt_info = vect_finish_replace_stmt (loop_vinfo, - scalar_dest_def_info, - new_stmt); + vect_finish_replace_stmt (loop_vinfo, + scalar_dest_def_info, + new_stmt); } else - new_stmt_info = vect_finish_stmt_generation (loop_vinfo, - scalar_dest_def_info, - new_stmt, gsi); + vect_finish_stmt_generation (loop_vinfo, + scalar_dest_def_info, + new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); - } - - if (!slp_node) - { - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); - *vec_stmt = new_stmt_info; + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + else + { + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + *vec_stmt = new_stmt; + } } return true; @@ -6822,7 +6811,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, bool vect_transform_reduction (loop_vec_info loop_vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node) + gimple **vec_stmt, slp_tree slp_node) { tree vectype_out = STMT_VINFO_VECTYPE (stmt_info); class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); @@ -6883,7 +6872,6 @@ vect_transform_reduction (loop_vec_info loop_vinfo, bool mask_by_cond_expr = use_mask_by_cond_expr_p (code, cond_fn, vectype_in); /* Transform. */ - stmt_vec_info new_stmt_info = NULL; tree new_temp = NULL_TREE; auto_vec<tree> vec_oprnds0; auto_vec<tree> vec_oprnds1; @@ -6938,6 +6926,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo, FOR_EACH_VEC_ELT (vec_oprnds0, i, def0) { + gimple *new_stmt; tree vop[3] = { def0, vec_oprnds1[i], NULL_TREE }; if (masked_loop_p && !mask_by_cond_expr) { @@ -6954,8 +6943,8 @@ vect_transform_reduction (loop_vec_info loop_vinfo, new_temp = make_ssa_name (vec_dest, call); gimple_call_set_lhs (call, new_temp); gimple_call_set_nothrow (call, true); - new_stmt_info = vect_finish_stmt_generation (loop_vinfo, - stmt_info, call, gsi); + vect_finish_stmt_generation (loop_vinfo, stmt_info, call, gsi); + new_stmt = call; } else { @@ -6969,28 +6958,27 @@ vect_transform_reduction (loop_vec_info loop_vinfo, build_vect_cond_expr (code, vop, mask, gsi); } - gassign *new_stmt = gimple_build_assign (vec_dest, code, - vop[0], vop[1], vop[2]); + new_stmt = gimple_build_assign (vec_dest, code, + vop[0], vop[1], vop[2]); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info = vect_finish_stmt_generation (loop_vinfo, stmt_info, - new_stmt, gsi); + vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi); } if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else if (single_defuse_cycle && i < ncopies - 1) { if (reduc_index == 0) - vec_oprnds0.safe_push (gimple_get_lhs (new_stmt_info->stmt)); + vec_oprnds0.safe_push (gimple_get_lhs (new_stmt)); else if (reduc_index == 1) - vec_oprnds1.safe_push (gimple_get_lhs (new_stmt_info->stmt)); + vec_oprnds1.safe_push (gimple_get_lhs (new_stmt)); else if (reduc_index == 2) - vec_oprnds2.safe_push (gimple_get_lhs (new_stmt_info->stmt)); + vec_oprnds2.safe_push (gimple_get_lhs (new_stmt)); } else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -7003,7 +6991,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo, bool vect_transform_cycle_phi (loop_vec_info loop_vinfo, - stmt_vec_info stmt_info, stmt_vec_info *vec_stmt, + stmt_vec_info stmt_info, gimple **vec_stmt, slp_tree slp_node, slp_instance slp_node_instance) { tree vectype_out = STMT_VINFO_VECTYPE (stmt_info); @@ -7138,7 +7126,6 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo, /* Create the reduction-phi that defines the reduction operand. */ gphi *new_phi = create_phi_node (vec_dest, loop->header); - stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi); /* Set the loop-entry arg of the reduction-phi. */ if (j != 0 && nested_cycle) @@ -7149,12 +7136,12 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo, /* The loop-latch arg is set in epilogue processing. */ if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi); else { if (j == 0) - *vec_stmt = new_phi_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi_info); + *vec_stmt = new_phi; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi); } } } @@ -7166,7 +7153,7 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo, bool vectorizable_lc_phi (loop_vec_info loop_vinfo, - stmt_vec_info stmt_info, stmt_vec_info *vec_stmt, + stmt_vec_info stmt_info, gimple **vec_stmt, slp_tree slp_node) { if (!loop_vinfo @@ -7198,11 +7185,10 @@ vectorizable_lc_phi (loop_vec_info loop_vinfo, /* Create the vectorized LC PHI node. */ gphi *new_phi = create_phi_node (vec_dest, bb); add_phi_arg (new_phi, vec_oprnds[i], e, UNKNOWN_LOCATION); - stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi); } if (!slp_node) *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; @@ -7262,7 +7248,7 @@ bool vectorizable_induction (loop_vec_info loop_vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); @@ -7495,8 +7481,6 @@ vectorizable_induction (loop_vec_info loop_vinfo, /* Create the induction-phi that defines the induction-operand. */ vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_"); induction_phi = create_phi_node (vec_dest, iv_loop->header); - stmt_vec_info induction_phi_info - = loop_vinfo->add_stmt (induction_phi); induc_def = PHI_RESULT (induction_phi); /* Create the iv update inside the loop */ @@ -7505,7 +7489,6 @@ vectorizable_induction (loop_vec_info loop_vinfo, vec_def = gimple_build (&stmts, PLUS_EXPR, step_vectype, vec_def, vec_step); vec_def = gimple_convert (&stmts, vectype, vec_def); - loop_vinfo->add_stmt (SSA_NAME_DEF_STMT (vec_def)); gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT); /* Set the arguments of the phi node: */ @@ -7513,7 +7496,7 @@ vectorizable_induction (loop_vec_info loop_vinfo, add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop), UNKNOWN_LOCATION); - SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi); } /* Fill up to the number of vectors we need for the whole group. */ nivs = least_common_multiple (group_size, @@ -7545,7 +7528,7 @@ vectorizable_induction (loop_vec_info loop_vinfo, step_vectype, NULL); for (; ivn < nvects; ++ivn) { - gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]->stmt; + gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]; tree def; if (gimple_code (iv) == GIMPLE_PHI) def = gimple_phi_result (iv); @@ -7563,8 +7546,8 @@ vectorizable_induction (loop_vec_info loop_vinfo, gimple_stmt_iterator tgsi = gsi_for_stmt (iv); gsi_insert_seq_after (&tgsi, stmts, GSI_CONTINUE_LINKING); } - SLP_TREE_VEC_STMTS (slp_node).quick_push - (loop_vinfo->add_stmt (SSA_NAME_DEF_STMT (def))); + SLP_TREE_VEC_STMTS (slp_node) + .quick_push (SSA_NAME_DEF_STMT (def)); } } @@ -7595,7 +7578,6 @@ vectorizable_induction (loop_vec_info loop_vinfo, new_bb = gsi_insert_on_edge_immediate (loop_preheader_edge (iv_loop), new_stmt); gcc_assert (!new_bb); - loop_vinfo->add_stmt (new_stmt); } } else @@ -7701,7 +7683,6 @@ vectorizable_induction (loop_vec_info loop_vinfo, /* Create the induction-phi that defines the induction-operand. */ vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_"); induction_phi = create_phi_node (vec_dest, iv_loop->header); - stmt_vec_info induction_phi_info = loop_vinfo->add_stmt (induction_phi); induc_def = PHI_RESULT (induction_phi); /* Create the iv update inside the loop */ @@ -7711,15 +7692,14 @@ vectorizable_induction (loop_vec_info loop_vinfo, vec_def = gimple_convert (&stmts, vectype, vec_def); gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT); new_stmt = SSA_NAME_DEF_STMT (vec_def); - stmt_vec_info new_stmt_info = loop_vinfo->add_stmt (new_stmt); /* Set the arguments of the phi node: */ add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION); add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop), UNKNOWN_LOCATION); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (induction_phi_info); - *vec_stmt = induction_phi_info; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (induction_phi); + *vec_stmt = induction_phi; /* In case that vectorization factor (VF) is bigger than the number of elements that we can fit in a vectype (nunits), we have to generate @@ -7768,8 +7748,7 @@ vectorizable_induction (loop_vec_info loop_vinfo, gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT); new_stmt = SSA_NAME_DEF_STMT (vec_def); - new_stmt_info = loop_vinfo->add_stmt (new_stmt); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } } @@ -7944,7 +7923,7 @@ vectorizable_live_operation (loop_vec_info loop_vinfo, gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)); /* Get the correct slp vectorized stmt. */ - gimple *vec_stmt = SLP_TREE_VEC_STMTS (slp_node)[vec_entry]->stmt; + gimple *vec_stmt = SLP_TREE_VEC_STMTS (slp_node)[vec_entry]; if (gphi *phi = dyn_cast <gphi *> (vec_stmt)) vec_lhs = gimple_phi_result (phi); else @@ -7957,7 +7936,7 @@ vectorizable_live_operation (loop_vec_info loop_vinfo, else { /* For multiple copies, get the last copy. */ - vec_lhs = gimple_get_lhs (STMT_VINFO_VEC_STMTS (stmt_info).last ()->stmt); + vec_lhs = gimple_get_lhs (STMT_VINFO_VEC_STMTS (stmt_info).last ()); /* Get the last lane in the vector. */ bitstart = int_const_binop (MINUS_EXPR, vec_bitsize, bitsize); @@ -8683,6 +8662,7 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call) } else { + /* Ignore vector stmts created in the outer loop. */ stmt_info = loop_vinfo->lookup_stmt (stmt); /* vector stmts created in the outer-loop during vectorization of diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index a1fd673..930f47e 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -5503,4 +5503,7 @@ vect_pattern_recog (vec_info *vinfo) &vect_vect_recog_func_ptrs[j], stmt_info); } } + + /* After this no more add_stmt calls are allowed. */ + vinfo->stmt_vec_info_ro = true; } diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index 44944bf..0217a52 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -3666,7 +3666,7 @@ tree vect_get_slp_vect_def (slp_tree slp_node, unsigned i) { if (SLP_TREE_VEC_STMTS (slp_node).exists ()) - return gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[i]->stmt); + return gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[i]); else return SLP_TREE_VEC_DEFS (slp_node)[i]; } @@ -3680,9 +3680,9 @@ vect_get_slp_defs (slp_tree slp_node, vec<tree> *vec_defs) if (SLP_TREE_DEF_TYPE (slp_node) == vect_internal_def) { unsigned j; - stmt_vec_info vec_def_stmt_info; - FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), j, vec_def_stmt_info) - vec_defs->quick_push (gimple_get_lhs (vec_def_stmt_info->stmt)); + gimple *vec_def_stmt; + FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), j, vec_def_stmt) + vec_defs->quick_push (gimple_get_lhs (vec_def_stmt)); } else vec_defs->splice (SLP_TREE_VEC_DEFS (slp_node)); @@ -3880,7 +3880,7 @@ vect_transform_slp_perm_load (vec_info *vinfo, /* Generate the permute statement if necessary. */ tree first_vec = dr_chain[first_vec_index + ri]; tree second_vec = dr_chain[second_vec_index + ri]; - stmt_vec_info perm_stmt_info; + gimple *perm_stmt; if (! noop_p) { gassign *stmt = as_a <gassign *> (stmt_info->stmt); @@ -3888,23 +3888,20 @@ vect_transform_slp_perm_load (vec_info *vinfo, = vect_create_destination_var (gimple_assign_lhs (stmt), vectype); perm_dest = make_ssa_name (perm_dest); - gassign *perm_stmt + perm_stmt = gimple_build_assign (perm_dest, VEC_PERM_EXPR, first_vec, second_vec, mask_vec); - perm_stmt_info - = vect_finish_stmt_generation (vinfo, - stmt_info, perm_stmt, - gsi); + vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, + gsi); } else /* If mask was NULL_TREE generate the requested identity transform. */ - perm_stmt_info = vinfo->lookup_def (first_vec); + perm_stmt = SSA_NAME_DEF_STMT (first_vec); /* Store the vector statement in NODE. */ - SLP_TREE_VEC_STMTS (node)[vect_stmts_counter++] - = perm_stmt_info; + SLP_TREE_VEC_STMTS (node)[vect_stmts_counter++] = perm_stmt; } } @@ -3995,8 +3992,8 @@ vect_schedule_slp_instance (vec_info *vinfo, } if (ocode != ERROR_MARK) { - vec<stmt_vec_info> v0; - vec<stmt_vec_info> v1; + vec<gimple *> v0; + vec<gimple *> v1; unsigned j; tree tmask = NULL_TREE; vect_transform_stmt (vinfo, stmt_info, &si, node, instance); @@ -4037,11 +4034,11 @@ vect_schedule_slp_instance (vec_info *vinfo, gimple *vstmt; vstmt = gimple_build_assign (make_ssa_name (vectype), VEC_PERM_EXPR, - gimple_assign_lhs (v0[j]->stmt), - gimple_assign_lhs (v1[j]->stmt), + gimple_assign_lhs (v0[j]), + gimple_assign_lhs (v1[j]), tmask); - SLP_TREE_VEC_STMTS (node).quick_push - (vect_finish_stmt_generation (vinfo, stmt_info, vstmt, &si)); + vect_finish_stmt_generation (vinfo, stmt_info, vstmt, &si); + SLP_TREE_VEC_STMTS (node).quick_push (vstmt); } v0.release (); v1.release (); @@ -4109,12 +4106,12 @@ vectorize_slp_instance_root_stmt (slp_tree node, slp_instance instance) if (SLP_TREE_NUMBER_OF_VEC_STMTS (node) == 1) { - stmt_vec_info child_stmt_info; + gimple *child_stmt; int j; - FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt_info) + FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt) { - tree vect_lhs = gimple_get_lhs (child_stmt_info->stmt); + tree vect_lhs = gimple_get_lhs (child_stmt); tree root_lhs = gimple_get_lhs (instance->root_stmt->stmt); if (!useless_type_conversion_p (TREE_TYPE (root_lhs), TREE_TYPE (vect_lhs))) @@ -4127,16 +4124,16 @@ vectorize_slp_instance_root_stmt (slp_tree node, slp_instance instance) else if (SLP_TREE_NUMBER_OF_VEC_STMTS (node) > 1) { int nelts = SLP_TREE_NUMBER_OF_VEC_STMTS (node); - stmt_vec_info child_stmt_info; + gimple *child_stmt; int j; vec<constructor_elt, va_gc> *v; vec_alloc (v, nelts); - FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt_info) + FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt) { CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, - gimple_get_lhs (child_stmt_info->stmt)); + gimple_get_lhs (child_stmt)); } tree lhs = gimple_get_lhs (instance->root_stmt->stmt); tree rtype = TREE_TYPE (gimple_assign_rhs1 (instance->root_stmt->stmt)); diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index d9db2f0..686eb38 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -1478,7 +1478,7 @@ vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info stmt_vinfo, gcc_assert (STMT_VINFO_VEC_STMTS (def_stmt_info).length () == ncopies); for (unsigned i = 0; i < ncopies; ++i) vec_oprnds->quick_push (gimple_get_lhs - (STMT_VINFO_VEC_STMTS (def_stmt_info)[i]->stmt)); + (STMT_VINFO_VEC_STMTS (def_stmt_info)[i])); } } @@ -1540,13 +1540,10 @@ vect_get_vec_defs (vec_info *vinfo, stmt_vec_info stmt_info, slp_tree slp_node, vect_finish_stmt_generation. Set the location of the new statement and create and return a stmt_vec_info for it. */ -static stmt_vec_info -vect_finish_stmt_generation_1 (vec_info *vinfo, +static void +vect_finish_stmt_generation_1 (vec_info *, stmt_vec_info stmt_info, gimple *vec_stmt) { - stmt_vec_info vec_stmt_info = vinfo->add_stmt (vec_stmt); - vec_stmt_info->vector_stmt = 1; - if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: %G", vec_stmt); @@ -1563,15 +1560,13 @@ vect_finish_stmt_generation_1 (vec_info *vinfo, } else gcc_assert (!stmt_could_throw_p (cfun, vec_stmt)); - - return vec_stmt_info; } /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT, which sets the same scalar result as STMT_INFO did. Create and return a stmt_vec_info for VEC_STMT. */ -stmt_vec_info +void vect_finish_replace_stmt (vec_info *vinfo, stmt_vec_info stmt_info, gimple *vec_stmt) { @@ -1581,13 +1576,13 @@ vect_finish_replace_stmt (vec_info *vinfo, gimple_stmt_iterator gsi = gsi_for_stmt (scalar_stmt); gsi_replace (&gsi, vec_stmt, true); - return vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt); + vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt); } /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it before *GSI. Create and return a stmt_vec_info for VEC_STMT. */ -stmt_vec_info +void vect_finish_stmt_generation (vec_info *vinfo, stmt_vec_info stmt_info, gimple *vec_stmt, gimple_stmt_iterator *gsi) @@ -1621,7 +1616,7 @@ vect_finish_stmt_generation (vec_info *vinfo, } } gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT); - return vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt); + vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt); } /* We want to vectorize a call to combined function CFN with function @@ -2564,7 +2559,7 @@ vect_build_zero_merge_argument (vec_info *vinfo, static void vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, + gimple **vec_stmt, gather_scatter_info *gs_info, tree mask) { @@ -2661,7 +2656,6 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info, tree src_op = NULL_TREE; tree mask_op = NULL_TREE; tree prev_res = NULL_TREE; - stmt_vec_info prev_stmt_info = NULL; if (!mask) { @@ -2765,29 +2759,26 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info, } src_op = build_zero_cst (srctype); } - gcall *new_call = gimple_build_call (gs_info->decl, 5, src_op, ptr, op, - mask_arg, scale); + gimple *new_stmt = gimple_build_call (gs_info->decl, 5, src_op, ptr, op, + mask_arg, scale); - stmt_vec_info new_stmt_info; if (!useless_type_conversion_p (vectype, rettype)) { gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype), TYPE_VECTOR_SUBPARTS (rettype))); op = vect_get_new_ssa_name (rettype, vect_simple_var); - gimple_call_set_lhs (new_call, op); - vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi); + gimple_call_set_lhs (new_stmt, op); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); var = make_ssa_name (vec_dest); op = build1 (VIEW_CONVERT_EXPR, vectype, op); - gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } else { - var = make_ssa_name (vec_dest, new_call); - gimple_call_set_lhs (new_call, var); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi); + var = make_ssa_name (vec_dest, new_stmt); + gimple_call_set_lhs (new_stmt, var); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } if (modifier == NARROW) @@ -2799,14 +2790,12 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info, } var = permute_vec_elements (vinfo, prev_res, var, perm_mask, stmt_info, gsi); - new_stmt_info = loop_vinfo->lookup_def (var); + new_stmt = SSA_NAME_DEF_STMT (var); } - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); - if (prev_stmt_info == NULL) - *vec_stmt = new_stmt_info; - prev_stmt_info = new_stmt_info; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } + *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; } /* Prepare the base and offset in GS_INFO for vectorization. @@ -2905,7 +2894,7 @@ vect_get_data_ptr_increment (vec_info *vinfo, static bool vectorizable_bswap (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, slp_tree *slp_op, tree vectype_in, stmt_vector_for_cost *cost_vec) { @@ -2993,12 +2982,11 @@ vectorizable_bswap (vec_info *vinfo, tem = make_ssa_name (vectype); new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR, vectype, tem2)); - stmt_vec_info new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -3043,7 +3031,7 @@ simple_integer_narrowing (tree vectype_out, tree vectype_in, static bool vectorizable_call (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { gcall *stmt; @@ -3333,7 +3321,6 @@ vectorizable_call (vec_info *vinfo, bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo); - stmt_vec_info new_stmt_info = NULL; if (modifier == NONE || ifn != IFN_LAST) { tree prev_res = NULL_TREE; @@ -3359,6 +3346,7 @@ vectorizable_call (vec_info *vinfo, vec<tree> vec_oprndsk = vec_defs[k]; vargs[k] = vec_oprndsk[i]; } + gimple *new_stmt; if (modifier == NARROW) { /* We don't define any narrowing conditional functions @@ -3376,12 +3364,10 @@ vectorizable_call (vec_info *vinfo, continue; } new_temp = make_ssa_name (vec_dest); - gimple *new_stmt - = gimple_build_assign (new_temp, convert_code, - prev_res, half_res); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + new_stmt = gimple_build_assign (new_temp, convert_code, + prev_res, half_res); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); } else { @@ -3404,11 +3390,10 @@ vectorizable_call (vec_info *vinfo, new_temp = make_ssa_name (vec_dest, call); gimple_call_set_lhs (call, new_temp); gimple_call_set_nothrow (call, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; } - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); } continue; } @@ -3434,6 +3419,7 @@ vectorizable_call (vec_info *vinfo, vargs[mask_opno], gsi); } + gimple *new_stmt; if (cfn == CFN_GOMP_SIMD_LANE) { tree cst = build_index_vector (vectype_out, j * nunits_out, 1); @@ -3442,9 +3428,8 @@ vectorizable_call (vec_info *vinfo, gimple *init_stmt = gimple_build_assign (new_var, cst); vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL); new_temp = make_ssa_name (vec_dest); - gimple *new_stmt = gimple_build_assign (new_temp, new_var); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (new_temp, new_var); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } else if (modifier == NARROW) { @@ -3462,10 +3447,9 @@ vectorizable_call (vec_info *vinfo, continue; } new_temp = make_ssa_name (vec_dest); - gassign *new_stmt = gimple_build_assign (new_temp, convert_code, - prev_res, half_res); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (new_temp, convert_code, + prev_res, half_res); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } else { @@ -3477,13 +3461,13 @@ vectorizable_call (vec_info *vinfo, new_temp = make_ssa_name (vec_dest, call); gimple_call_set_lhs (call, new_temp); gimple_call_set_nothrow (call, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; } if (j == (modifier == NARROW ? 1 : 0)) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } for (i = 0; i < nargs; i++) { @@ -3530,9 +3514,8 @@ vectorizable_call (vec_info *vinfo, new_temp = make_ssa_name (vec_dest, call); gimple_call_set_lhs (call, new_temp); gimple_call_set_nothrow (call, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + SLP_TREE_VEC_STMTS (slp_node).quick_push (call); } continue; } @@ -3556,10 +3539,9 @@ vectorizable_call (vec_info *vinfo, gcall *new_stmt = gimple_build_call_vec (fndecl, vargs); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_call_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -3695,7 +3677,7 @@ simd_clone_subparts (tree vectype) static bool vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *) { tree vec_dest; @@ -4154,7 +4136,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, } tree phi_res = copy_ssa_name (op); gphi *new_phi = create_phi_node (phi_res, loop->header); - loop_vinfo->add_stmt (new_phi); add_phi_arg (new_phi, arginfo[i].op, loop_preheader_edge (loop), UNKNOWN_LOCATION); enum tree_code code @@ -4171,7 +4152,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, = gimple_build_assign (phi_arg, code, phi_res, tcst); gimple_stmt_iterator si = gsi_after_labels (loop->header); gsi_insert_after (&si, new_stmt, GSI_NEW_STMT); - loop_vinfo->add_stmt (new_stmt); add_phi_arg (new_phi, phi_arg, loop_latch_edge (loop), UNKNOWN_LOCATION); arginfo[i].op = phi_res; @@ -4220,8 +4200,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, new_temp = make_ssa_name (rtype, new_call); gimple_call_set_lhs (new_call, new_temp); } - stmt_vec_info new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi); + gimple *new_stmt = new_call; if (vec_dest) { @@ -4244,15 +4224,12 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, else t = build3 (BIT_FIELD_REF, vectype, new_temp, bitsize_int (prec), bitsize_int (l * prec)); - gimple *new_stmt - = gimple_build_assign (make_ssa_name (vectype), t); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + new_stmt = gimple_build_assign (make_ssa_name (vectype), t); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (j == 0 && l == 0) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (ratype) @@ -4273,11 +4250,10 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, { tree tem = build4 (ARRAY_REF, rtype, new_temp, size_int (m), NULL_TREE, NULL_TREE); - gimple *new_stmt - = gimple_build_assign (make_ssa_name (rtype), tem); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + new_stmt = gimple_build_assign (make_ssa_name (rtype), + tem); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE, gimple_assign_lhs (new_stmt)); } @@ -4288,14 +4264,13 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, if ((j & (k - 1)) != k - 1) continue; vec_oprnd0 = build_constructor (vectype, ret_ctor_elts); - gimple *new_stmt + new_stmt = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if ((unsigned) j == k - 1) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); continue; } else if (ratype) @@ -4303,17 +4278,15 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, tree t = build_fold_addr_expr (new_temp); t = build2 (MEM_REF, vectype, t, build_int_cst (TREE_TYPE (t), 0)); - gimple *new_stmt - = gimple_build_assign (make_ssa_name (vec_dest), t); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (make_ssa_name (vec_dest), t); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); vect_clobber_variable (vinfo, stmt_info, gsi, new_temp); } } if (j == 0) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } for (i = 0; i < nargs; ++i) @@ -4403,8 +4376,7 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds, gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1); new_tmp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_tmp); - stmt_vec_info new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (multi_step_cvt) /* Store the resulting vector for next recursive call. */ @@ -4415,9 +4387,9 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds, vectors in SLP_NODE or in vector info of the scalar statement (or in STMT_VINFO_RELATED_STMT chain). */ if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } } @@ -4503,7 +4475,7 @@ vect_create_vectorized_promotion_stmts (vec_info *vinfo, static bool vectorizable_conversion (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree vec_dest; @@ -4887,19 +4859,17 @@ vectorizable_conversion (vec_info *vinfo, op0, &vec_oprnds0); FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0) { - stmt_vec_info new_stmt_info; /* Arguments are ready, create the new vector stmt. */ gcc_assert (TREE_CODE_LENGTH (code1) == unary_op); gassign *new_stmt = gimple_build_assign (vec_dest, code1, vop0); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } break; @@ -4936,23 +4906,21 @@ vectorizable_conversion (vec_info *vinfo, FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0) { - stmt_vec_info new_stmt_info; + gimple *new_stmt; if (cvt_type) { gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op); new_temp = make_ssa_name (vec_dest); - gassign *new_stmt - = gimple_build_assign (new_temp, codecvt1, vop0); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (new_temp, codecvt1, vop0); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } else - new_stmt_info = vinfo->lookup_def (vop0); + new_stmt = SSA_NAME_DEF_STMT (vop0); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } break; @@ -5025,7 +4993,7 @@ vect_nop_conversion_p (stmt_vec_info stmt_info) static bool vectorizable_assignment (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree vec_dest; @@ -5155,7 +5123,6 @@ vectorizable_assignment (vec_info *vinfo, vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, op, &vec_oprnds); /* Arguments are ready. create the new vector stmt. */ - stmt_vec_info new_stmt_info = NULL; FOR_EACH_VEC_ELT (vec_oprnds, i, vop) { if (CONVERT_EXPR_CODE_P (code) @@ -5164,12 +5131,11 @@ vectorizable_assignment (vec_info *vinfo, gassign *new_stmt = gimple_build_assign (vec_dest, vop); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; @@ -5225,7 +5191,7 @@ vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type) static bool vectorizable_shift (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree vec_dest; @@ -5606,19 +5572,17 @@ vectorizable_shift (vec_info *vinfo, vec_oprnd1 ? NULL_TREE : op1, &vec_oprnds1); /* Arguments are ready. Create the new vector stmt. */ - stmt_vec_info new_stmt_info = NULL; FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0) { vop1 = vec_oprnds1[i]; gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -5642,7 +5606,7 @@ vectorizable_shift (vec_info *vinfo, static bool vectorizable_operation (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree vec_dest; @@ -6015,9 +5979,9 @@ vectorizable_operation (vec_info *vinfo, vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, op0, &vec_oprnds0, op1, &vec_oprnds1, op2, &vec_oprnds2); /* Arguments are ready. Create the new vector stmt. */ - stmt_vec_info new_stmt_info = NULL; FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0) { + gimple *new_stmt = NULL; vop1 = ((op_type == binary_op || op_type == ternary_op) ? vec_oprnds1[i] : NULL_TREE); vop2 = ((op_type == ternary_op) ? vec_oprnds2[i] : NULL_TREE); @@ -6034,33 +5998,30 @@ vectorizable_operation (vec_info *vinfo, new_temp = make_ssa_name (vec_dest, call); gimple_call_set_lhs (call, new_temp); gimple_call_set_nothrow (call, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; } else { - gassign *new_stmt = gimple_build_assign (vec_dest, code, - vop0, vop1, vop2); + new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1, vop2); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (vec_cvt_dest) { new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp); - gassign *new_stmt - = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR, - new_temp); + new_stmt = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR, + new_temp); new_temp = make_ssa_name (vec_cvt_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); } } if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -6723,7 +6684,7 @@ check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype, static bool vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, int ncopies) + gimple **vec_stmt, int ncopies) { loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info); @@ -6843,7 +6804,6 @@ vectorizable_scan_store (vec_info *vinfo, rhs2, &vec_oprnds3); for (int j = 0; j < ncopies; j++) { - stmt_vec_info new_stmt_info; vec_oprnd1 = vec_oprnds1[j]; if (ldataref_ptr == NULL) vec_oprnd2 = vec_oprnds2[j]; @@ -6861,8 +6821,8 @@ vectorizable_scan_store (vec_info *vinfo, dataref_offset); vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr)); gimple *g = gimple_build_assign (vec_oprnd2, data_ref); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; } @@ -6876,8 +6836,8 @@ vectorizable_scan_store (vec_info *vinfo, != scan_store_kind_perm)) ? zero_vec : vec_oprnd1, v, perms[i]); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond) @@ -6893,9 +6853,9 @@ vectorizable_scan_store (vec_info *vinfo, tree new_temp2 = make_ssa_name (vectype); g = gimple_build_assign (new_temp2, VEC_COND_EXPR, vb.build (), new_temp, vec_oprnd1); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); new_temp = new_temp2; } @@ -6912,16 +6872,16 @@ vectorizable_scan_store (vec_info *vinfo, tree new_temp2 = make_ssa_name (vectype); g = gimple_build_assign (new_temp2, code, v, new_temp); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); v = new_temp2; } tree new_temp = make_ssa_name (vectype); gimple *g = gimple_build_assign (new_temp, code, orig, v); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); tree last_perm_arg = new_temp; /* For exclusive scan, new_temp computed above is the exclusive scan @@ -6931,15 +6891,15 @@ vectorizable_scan_store (vec_info *vinfo, { last_perm_arg = make_ssa_name (vectype); g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); } orig = make_ssa_name (vectype); g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg, last_perm_arg, perms[units_log2]); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); if (!inscan_var_store) { @@ -6948,8 +6908,8 @@ vectorizable_scan_store (vec_info *vinfo, dataref_offset); vect_copy_ref_info (data_ref, DR_REF (dr_info->dr)); g = gimple_build_assign (data_ref, new_temp); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); } } @@ -6964,9 +6924,8 @@ vectorizable_scan_store (vec_info *vinfo, dataref_offset); vect_copy_ref_info (data_ref, DR_REF (dr_info->dr)); gimple *g = gimple_build_assign (data_ref, orig); - stmt_vec_info new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); } return true; } @@ -6983,7 +6942,7 @@ vectorizable_scan_store (vec_info *vinfo, static bool vectorizable_store (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree data_ref; @@ -7399,10 +7358,9 @@ vectorizable_store (vec_info *vinfo, gcall *new_stmt = gimple_build_call (gs_info.decl, 5, ptr, mask_arg, op, src, scale); - stmt_vec_info new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; return true; @@ -7580,7 +7538,6 @@ vectorizable_store (vec_info *vinfo, loop, &incr_gsi, insert_after, &offvar, NULL); incr = gsi_stmt (incr_gsi); - loop_vinfo->add_stmt (incr); stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step); @@ -7643,9 +7600,7 @@ vectorizable_store (vec_info *vinfo, /* And store it to *running_off. */ assign = gimple_build_assign (newref, elem); - stmt_vec_info assign_info - = vect_finish_stmt_generation (vinfo, stmt_info, - assign, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, assign, gsi); group_el += lnel; if (! slp @@ -7663,8 +7618,8 @@ vectorizable_store (vec_info *vinfo, && !slp) { if (j == 0 && i == 0) - *vec_stmt = assign_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (assign_info); + *vec_stmt = assign; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (assign); } } } @@ -7777,7 +7732,7 @@ vectorizable_store (vec_info *vinfo, gvec_oprnds.safe_grow_cleared (group_size); for (j = 0; j < ncopies; j++) { - stmt_vec_info new_stmt_info; + gimple *new_stmt; if (j == 0) { if (slp) @@ -7924,15 +7879,15 @@ vectorizable_store (vec_info *vinfo, gimple_call_set_lhs (call, data_ref); } gimple_call_set_nothrow (call, true); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, - call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; /* Record that VEC_ARRAY is now dead. */ vect_clobber_variable (vinfo, stmt_info, gsi, vec_array); } else { - new_stmt_info = NULL; + new_stmt = NULL; if (grouped_store) { if (j == 0) @@ -7970,8 +7925,8 @@ vectorizable_store (vec_info *vinfo, (IFN_SCATTER_STORE, 4, dataref_ptr, vec_offset, scale, vec_oprnd); gimple_call_set_nothrow (call, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; break; } @@ -8029,8 +7984,8 @@ vectorizable_store (vec_info *vinfo, dataref_ptr, ptr, final_mask, vec_oprnd); gimple_call_set_nothrow (call, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; } else { @@ -8050,10 +8005,8 @@ vectorizable_store (vec_info *vinfo, = build_aligned_type (TREE_TYPE (data_ref), TYPE_ALIGN (elem_type)); vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); - gassign *new_stmt - = gimple_build_assign (data_ref, vec_oprnd); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (data_ref, vec_oprnd); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } if (slp) @@ -8067,8 +8020,8 @@ vectorizable_store (vec_info *vinfo, if (!slp) { if (j == 0) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } } @@ -8205,7 +8158,7 @@ hoist_defs_of_uses (stmt_vec_info stmt_info, class loop *loop) static bool vectorizable_load (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree scalar_dest; @@ -8535,27 +8488,20 @@ vectorizable_load (vec_info *vinfo, gsi_next (&gsi2); for (j = 0; j < ncopies; j++) { - stmt_vec_info new_stmt_info; if (hoist_p) - { - new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest, - vectype, NULL); - gimple *new_stmt = SSA_NAME_DEF_STMT (new_temp); - new_stmt_info = vinfo->add_stmt (new_stmt); - } + new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest, + vectype, NULL); else - { - new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest, - vectype, &gsi2); - new_stmt_info = vinfo->lookup_def (new_temp); - } + new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest, + vectype, &gsi2); + gimple *new_stmt = SSA_NAME_DEF_STMT (new_temp); if (slp) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else { if (j == 0) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } } return true; @@ -8566,7 +8512,6 @@ vectorizable_load (vec_info *vinfo, { gimple_stmt_iterator incr_gsi; bool insert_after; - gimple *incr; tree offvar; tree ivstep; tree running_off; @@ -8641,8 +8586,6 @@ vectorizable_load (vec_info *vinfo, create_iv (stride_base, ivstep, NULL, loop, &incr_gsi, insert_after, &offvar, NULL); - incr = gsi_stmt (incr_gsi); - loop_vinfo->add_stmt (incr); stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step); @@ -8712,17 +8655,15 @@ vectorizable_load (vec_info *vinfo, { if (nloads > 1) vec_alloc (v, nloads); - stmt_vec_info new_stmt_info = NULL; + gimple *new_stmt = NULL; for (i = 0; i < nloads; i++) { tree this_off = build_int_cst (TREE_TYPE (alias_off), group_el * elsz + cst_offset); tree data_ref = build2 (MEM_REF, ltype, running_off, this_off); vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); - gassign *new_stmt - = gimple_build_assign (make_ssa_name (ltype), data_ref); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (make_ssa_name (ltype), data_ref); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (nloads > 1) CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, gimple_assign_lhs (new_stmt)); @@ -8745,32 +8686,29 @@ vectorizable_load (vec_info *vinfo, tree vec_inv = build_constructor (lvectype, v); new_temp = vect_init_vector (vinfo, stmt_info, vec_inv, lvectype, gsi); - new_stmt_info = vinfo->lookup_def (new_temp); + new_stmt = SSA_NAME_DEF_STMT (new_temp); if (lvectype != vectype) { - gassign *new_stmt - = gimple_build_assign (make_ssa_name (vectype), - VIEW_CONVERT_EXPR, - build1 (VIEW_CONVERT_EXPR, - vectype, new_temp)); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + new_stmt = gimple_build_assign (make_ssa_name (vectype), + VIEW_CONVERT_EXPR, + build1 (VIEW_CONVERT_EXPR, + vectype, new_temp)); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } } if (slp) { if (slp_perm) - dr_chain.quick_push (gimple_assign_lhs (new_stmt_info->stmt)); + dr_chain.quick_push (gimple_assign_lhs (new_stmt)); else - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); } else { if (j == 0) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } } if (slp_perm) @@ -9052,7 +8990,6 @@ vectorizable_load (vec_info *vinfo, poly_uint64 group_elt = 0; for (j = 0; j < ncopies; j++) { - stmt_vec_info new_stmt_info = NULL; /* 1. Create the vector or array pointer update chain. */ if (j == 0) { @@ -9131,6 +9068,7 @@ vectorizable_load (vec_info *vinfo, if (grouped_load || slp_perm) dr_chain.create (vec_num); + gimple *new_stmt = NULL; if (memory_access_type == VMAT_LOAD_STORE_LANES) { tree vec_array; @@ -9166,8 +9104,8 @@ vectorizable_load (vec_info *vinfo, } gimple_call_set_lhs (call, vec_array); gimple_call_set_nothrow (call, true); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, - call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; /* Extract each vector into an SSA_NAME. */ for (i = 0; i < vec_num; i++) @@ -9202,7 +9140,6 @@ vectorizable_load (vec_info *vinfo, gsi, stmt_info, bump); /* 2. Create the vector-load in the loop. */ - gimple *new_stmt = NULL; switch (alignment_support_scheme) { case dr_aligned: @@ -9448,9 +9385,7 @@ vectorizable_load (vec_info *vinfo, } new_temp = make_ssa_name (vec_dest, new_stmt); gimple_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); /* 3. Handle explicit realignment if necessary/supported. Create in loop: @@ -9466,9 +9401,7 @@ vectorizable_load (vec_info *vinfo, msq, lsq, realignment_token); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (alignment_support_scheme == dr_explicit_realign_optimized) { @@ -9486,7 +9419,7 @@ vectorizable_load (vec_info *vinfo, tree perm_mask = perm_mask_for_reverse (vectype); new_temp = permute_vec_elements (vinfo, new_temp, new_temp, perm_mask, stmt_info, gsi); - new_stmt_info = vinfo->lookup_def (new_temp); + new_stmt = SSA_NAME_DEF_STMT (new_temp); } /* Collect vector loads and later create their permutation in @@ -9496,7 +9429,7 @@ vectorizable_load (vec_info *vinfo, /* Store vector loads in the corresponding SLP_NODE. */ if (slp && !slp_perm) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); /* With SLP permutation we load the gaps as well, without we need to skip the gaps after we manage to fully load @@ -9549,13 +9482,13 @@ vectorizable_load (vec_info *vinfo, } else { - if (j == 0) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } } dr_chain.release (); } + if (!slp) + *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; return true; } @@ -9667,7 +9600,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info, static bool vectorizable_condition (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree scalar_dest = NULL_TREE; @@ -10090,19 +10023,18 @@ vectorizable_condition (vec_info *vinfo, } } - stmt_vec_info new_stmt_info; + gimple *new_stmt; if (reduction_type == EXTRACT_LAST_REDUCTION) { gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt; tree lhs = gimple_get_lhs (old_stmt); - gcall *new_stmt = gimple_build_call_internal + new_stmt = gimple_build_call_internal (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare, vec_then_clause); gimple_call_set_lhs (new_stmt, lhs); SSA_NAME_DEF_STMT (lhs) = new_stmt; if (old_stmt == gsi_stmt (*gsi)) - new_stmt_info = vect_finish_replace_stmt (vinfo, - stmt_info, new_stmt); + vect_finish_replace_stmt (vinfo, stmt_info, new_stmt); else { /* In this case we're moving the definition to later in the @@ -10110,23 +10042,20 @@ vectorizable_condition (vec_info *vinfo, lhs are in phi statements. */ gimple_stmt_iterator old_gsi = gsi_for_stmt (old_stmt); gsi_remove (&old_gsi, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } } else { new_temp = make_ssa_name (vec_dest); - gassign *new_stmt - = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare, - vec_then_clause, vec_else_clause); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare, + vec_then_clause, vec_else_clause); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -10151,7 +10080,7 @@ vectorizable_condition (vec_info *vinfo, static bool vectorizable_comparison (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree lhs, rhs1, rhs2; @@ -10344,27 +10273,24 @@ vectorizable_comparison (vec_info *vinfo, /* Arguments are ready. Create the new vector stmt. */ FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1) { - stmt_vec_info new_stmt_info; + gimple *new_stmt; vec_rhs2 = vec_oprnds1[i]; new_temp = make_ssa_name (mask); if (bitop1 == NOP_EXPR) { - gassign *new_stmt = gimple_build_assign (new_temp, code, - vec_rhs1, vec_rhs2); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (new_temp, code, + vec_rhs1, vec_rhs2); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } else { - gassign *new_stmt; if (bitop1 == BIT_NOT_EXPR) new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs2); else new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1, vec_rhs2); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (bitop2 != NOP_EXPR) { tree res = make_ssa_name (mask); @@ -10373,14 +10299,13 @@ vectorizable_comparison (vec_info *vinfo, else new_stmt = gimple_build_assign (res, bitop2, vec_rhs1, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } } if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -10666,7 +10591,7 @@ vect_transform_stmt (vec_info *vinfo, slp_tree slp_node, slp_instance slp_node_instance) { bool is_store = false; - stmt_vec_info vec_stmt = NULL; + gimple *vec_stmt = NULL; bool done; gcc_assert (slp_node || !PURE_SLP_STMT (stmt_info)); @@ -10808,14 +10733,14 @@ vect_transform_stmt (vec_info *vinfo, && (PHI_ARG_DEF_FROM_EDGE (phi, e) == gimple_get_lhs (orig_stmt_info->stmt))) { - vec<stmt_vec_info> &phi_info + vec<gimple *> &phi_info = STMT_VINFO_VEC_STMTS (STMT_VINFO_REDUC_DEF (orig_stmt_info)); - vec<stmt_vec_info> &vec_stmt + vec<gimple *> &vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info); gcc_assert (phi_info.length () == vec_stmt.length ()); for (unsigned i = 0; i < phi_info.length (); ++i) - add_phi_arg (as_a <gphi *> (phi_info[i]->stmt), - gimple_get_lhs (vec_stmt[i]->stmt), e, + add_phi_arg (as_a <gphi *> (phi_info[i]), + gimple_get_lhs (vec_stmt[i]), e, gimple_phi_arg_location (phi, e->dest_idx)); } else if (slp_node @@ -10827,7 +10752,7 @@ vect_transform_stmt (vec_info *vinfo, gcc_assert (SLP_TREE_VEC_STMTS (phi_node).length () == SLP_TREE_VEC_STMTS (slp_node).length ()); for (unsigned i = 0; i < SLP_TREE_VEC_STMTS (phi_node).length (); ++i) - add_phi_arg (as_a <gphi *> (SLP_TREE_VEC_STMTS (phi_node)[i]->stmt), + add_phi_arg (as_a <gphi *> (SLP_TREE_VEC_STMTS (phi_node)[i]), vect_get_slp_vect_def (slp_node, i), e, gimple_phi_arg_location (phi, e->dest_idx)); } diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c index e9fe384..76cfba5 100644 --- a/gcc/tree-vectorizer.c +++ b/gcc/tree-vectorizer.c @@ -461,6 +461,7 @@ vec_info::vec_info (vec_info::vec_kind kind_in, void *target_cost_data_in, vec_info_shared *shared_) : kind (kind_in), shared (shared_), + stmt_vec_info_ro (false), target_cost_data (target_cost_data_in) { stmt_vec_infos.create (50); @@ -619,8 +620,7 @@ vec_info::replace_stmt (gimple_stmt_iterator *gsi, stmt_vec_info stmt_info, { gimple *old_stmt = stmt_info->stmt; gcc_assert (!stmt_info->pattern_stmt_p && old_stmt == gsi_stmt (*gsi)); - set_vinfo_for_stmt (old_stmt, NULL); - set_vinfo_for_stmt (new_stmt, stmt_info); + gimple_set_uid (new_stmt, gimple_uid (old_stmt)); stmt_info->stmt = new_stmt; gsi_replace (gsi, new_stmt, true); } @@ -666,6 +666,7 @@ vec_info::set_vinfo_for_stmt (gimple *stmt, stmt_vec_info info) unsigned int uid = gimple_uid (stmt); if (uid == 0) { + gcc_assert (!stmt_vec_info_ro); gcc_checking_assert (info); uid = stmt_vec_infos.length () + 1; gimple_set_uid (stmt, uid); diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index df05638..14f68f7 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -138,7 +138,7 @@ struct _slp_tree { tree vectype; /* Vectorized stmt/s. */ - vec<stmt_vec_info> vec_stmts; + vec<gimple *> vec_stmts; vec<tree> vec_defs; /* Number of vector stmts that are created to replace the group of scalar stmts. It is calculated during the transformation phase as the number of @@ -338,6 +338,8 @@ public: /* The mapping of GIMPLE UID to stmt_vec_info. */ vec<stmt_vec_info> stmt_vec_infos; + /* Whether the above mapping is complete. */ + bool stmt_vec_info_ro; /* The SLP graph. */ auto_vec<slp_instance> slp_instances; @@ -965,7 +967,7 @@ public: tree vectype; /* The vectorized stmts. */ - vec<stmt_vec_info> vec_stmts; + vec<gimple *> vec_stmts; /* The following is relevant only for stmts that contain a non-scalar data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have @@ -1126,8 +1128,6 @@ public: /* True if this is only suitable for SLP vectorization. */ bool slp_vect_only_p; - - bool vector_stmt; }; /* Information about a gather/scatter call. */ @@ -1169,7 +1169,7 @@ struct gather_scatter_info { #define STMT_VINFO_RELEVANT(S) (S)->relevant #define STMT_VINFO_LIVE_P(S) (S)->live #define STMT_VINFO_VECTYPE(S) (S)->vectype -#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts +#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts #define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable #define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0) #define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p @@ -1737,11 +1737,9 @@ record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count, STMT_VINFO_VECTYPE (stmt_info), misalign, where); } -extern stmt_vec_info vect_finish_replace_stmt (vec_info *, - stmt_vec_info, gimple *); -extern stmt_vec_info vect_finish_stmt_generation (vec_info *, - stmt_vec_info, gimple *, - gimple_stmt_iterator *); +extern void vect_finish_replace_stmt (vec_info *, stmt_vec_info, gimple *); +extern void vect_finish_stmt_generation (vec_info *, stmt_vec_info, gimple *, + gimple_stmt_iterator *); extern opt_result vect_mark_stmts_to_be_vectorized (loop_vec_info, bool *); extern tree vect_get_store_rhs (stmt_vec_info); void vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info, unsigned, @@ -1872,16 +1870,16 @@ extern bool vectorizable_reduction (loop_vec_info, stmt_vec_info, stmt_vector_for_cost *); extern bool vectorizable_induction (loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, - stmt_vec_info *, slp_tree, + gimple **, slp_tree, stmt_vector_for_cost *); extern bool vect_transform_reduction (loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, - stmt_vec_info *, slp_tree); + gimple **, slp_tree); extern bool vect_transform_cycle_phi (loop_vec_info, stmt_vec_info, - stmt_vec_info *, + gimple **, slp_tree, slp_instance); extern bool vectorizable_lc_phi (loop_vec_info, stmt_vec_info, - stmt_vec_info *, slp_tree); + gimple **, slp_tree); extern bool vect_worthwhile_without_simd_p (vec_info *, tree_code); extern int vect_get_known_peeling_cost (loop_vec_info, int, int *, stmt_vector_for_cost *, |