diff options
Diffstat (limited to 'gcc/tree-vect-loop-manip.c')
| -rw-r--r-- | gcc/tree-vect-loop-manip.c | 96 |
1 files changed, 89 insertions, 7 deletions
diff --git a/gcc/tree-vect-loop-manip.c b/gcc/tree-vect-loop-manip.c index 7cf00e6..2370b87 100644 --- a/gcc/tree-vect-loop-manip.c +++ b/gcc/tree-vect-loop-manip.c @@ -1084,6 +1084,33 @@ slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop, exit = single_exit (loop); basic_block new_preheader = new_bbs[0]; + /* Before installing PHI arguments make sure that the edges + into them match that of the scalar loop we analyzed. This + makes sure the SLP tree matches up between the main vectorized + loop and the epilogue vectorized copies. */ + if (single_succ_edge (preheader)->dest_idx + != single_succ_edge (new_bbs[0])->dest_idx) + { + basic_block swap_bb = new_bbs[1]; + gcc_assert (EDGE_COUNT (swap_bb->preds) == 2); + std::swap (EDGE_PRED (swap_bb, 0), EDGE_PRED (swap_bb, 1)); + EDGE_PRED (swap_bb, 0)->dest_idx = 0; + EDGE_PRED (swap_bb, 1)->dest_idx = 1; + } + if (duplicate_outer_loop) + { + class loop *new_inner_loop = get_loop_copy (scalar_loop->inner); + if (loop_preheader_edge (scalar_loop)->dest_idx + != loop_preheader_edge (new_inner_loop)->dest_idx) + { + basic_block swap_bb = new_inner_loop->header; + gcc_assert (EDGE_COUNT (swap_bb->preds) == 2); + std::swap (EDGE_PRED (swap_bb, 0), EDGE_PRED (swap_bb, 1)); + EDGE_PRED (swap_bb, 0)->dest_idx = 0; + EDGE_PRED (swap_bb, 1)->dest_idx = 1; + } + } + add_phi_args_after_copy (new_bbs, scalar_loop->num_nodes + 1, NULL); /* Skip new preheader since it's deleted if copy loop is added at entry. */ @@ -2007,13 +2034,29 @@ vect_gen_vector_loop_niters (loop_vec_info loop_vinfo, tree niters, niters_vector = force_gimple_operand (niters_vector, &stmts, true, var); gsi_insert_seq_on_edge_immediate (pe, stmts); /* Peeling algorithm guarantees that vector loop bound is at least ONE, - we set range information to make niters analyzer's life easier. */ + we set range information to make niters analyzer's life easier. + Note the number of latch iteration value can be TYPE_MAX_VALUE so + we have to represent the vector niter TYPE_MAX_VALUE + 1 >> log_vf. */ if (stmts != NULL && log_vf) - set_range_info (niters_vector, VR_RANGE, - wi::to_wide (build_int_cst (type, 1)), - wi::to_wide (fold_build2 (RSHIFT_EXPR, type, - TYPE_MAX_VALUE (type), - log_vf))); + { + if (niters_no_overflow) + set_range_info (niters_vector, VR_RANGE, + wi::one (TYPE_PRECISION (type)), + wi::rshift (wi::max_value (TYPE_PRECISION (type), + TYPE_SIGN (type)), + exact_log2 (const_vf), + TYPE_SIGN (type))); + /* For VF == 1 the vector IV might also overflow so we cannot + assert a minimum value of 1. */ + else if (const_vf > 1) + set_range_info (niters_vector, VR_RANGE, + wi::one (TYPE_PRECISION (type)), + wi::rshift (wi::max_value (TYPE_PRECISION (type), + TYPE_SIGN (type)) + - (const_vf - 1), + exact_log2 (const_vf), TYPE_SIGN (type)) + + 1); + } } *niters_vector_ptr = niters_vector; *step_vector_ptr = step_vector; @@ -2545,6 +2588,45 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, if (!prolog_peeling && !epilog_peeling) return NULL; + /* Before doing any peeling make sure to reset debug binds outside of + the loop refering to defs not in LC SSA. */ + class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); + for (unsigned i = 0; i < loop->num_nodes; ++i) + { + basic_block bb = LOOP_VINFO_BBS (loop_vinfo)[i]; + imm_use_iterator ui; + gimple *use_stmt; + for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi); + gsi_next (&gsi)) + { + FOR_EACH_IMM_USE_STMT (use_stmt, ui, gimple_phi_result (gsi.phi ())) + if (gimple_debug_bind_p (use_stmt) + && loop != gimple_bb (use_stmt)->loop_father + && !flow_loop_nested_p (loop, + gimple_bb (use_stmt)->loop_father)) + { + gimple_debug_bind_reset_value (use_stmt); + update_stmt (use_stmt); + } + } + for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi); + gsi_next (&gsi)) + { + ssa_op_iter op_iter; + def_operand_p def_p; + FOR_EACH_SSA_DEF_OPERAND (def_p, gsi_stmt (gsi), op_iter, SSA_OP_DEF) + FOR_EACH_IMM_USE_STMT (use_stmt, ui, DEF_FROM_PTR (def_p)) + if (gimple_debug_bind_p (use_stmt) + && loop != gimple_bb (use_stmt)->loop_father + && !flow_loop_nested_p (loop, + gimple_bb (use_stmt)->loop_father)) + { + gimple_debug_bind_reset_value (use_stmt); + update_stmt (use_stmt); + } + } + } + prob_vector = profile_probability::guessed_always ().apply_scale (9, 10); estimated_vf = vect_vf_for_cost (loop_vinfo); if (estimated_vf == 2) @@ -2552,7 +2634,7 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, prob_prolog = prob_epilog = profile_probability::guessed_always () .apply_scale (estimated_vf - 1, estimated_vf); - class loop *prolog, *epilog = NULL, *loop = LOOP_VINFO_LOOP (loop_vinfo); + class loop *prolog, *epilog = NULL; class loop *first_loop = loop; bool irred_flag = loop_preheader_edge (loop)->flags & EDGE_IRREDUCIBLE_LOOP; |