diff options
Diffstat (limited to 'gcc/tree-vect-loop-manip.cc')
| -rw-r--r-- | gcc/tree-vect-loop-manip.cc | 136 |
1 files changed, 124 insertions, 12 deletions
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc index ff6159e..acf3642 100644 --- a/gcc/tree-vect-loop-manip.cc +++ b/gcc/tree-vect-loop-manip.cc @@ -385,6 +385,63 @@ vect_maybe_permute_loop_masks (gimple_seq *seq, rgroup_controls *dest_rgm, return false; } +/* Populate DEST_RGM->controls, given that they should add up to STEP. + + STEP = MIN_EXPR <ivtmp_34, VF>; + + First length (MIN (X, VF/N)): + loop_len_15 = MIN_EXPR <STEP, VF/N>; + + Second length: + tmp = STEP - loop_len_15; + loop_len_16 = MIN (tmp, VF/N); + + Third length: + tmp2 = tmp - loop_len_16; + loop_len_17 = MIN (tmp2, VF/N); + + Last length: + loop_len_18 = tmp2 - loop_len_17; +*/ + +static void +vect_adjust_loop_lens_control (tree iv_type, gimple_seq *seq, + rgroup_controls *dest_rgm, tree step) +{ + tree ctrl_type = dest_rgm->type; + poly_uint64 nitems_per_ctrl + = TYPE_VECTOR_SUBPARTS (ctrl_type) * dest_rgm->factor; + tree length_limit = build_int_cst (iv_type, nitems_per_ctrl); + + for (unsigned int i = 0; i < dest_rgm->controls.length (); ++i) + { + tree ctrl = dest_rgm->controls[i]; + if (i == 0) + { + /* First iteration: MIN (X, VF/N) capped to the range [0, VF/N]. */ + gassign *assign + = gimple_build_assign (ctrl, MIN_EXPR, step, length_limit); + gimple_seq_add_stmt (seq, assign); + } + else if (i == dest_rgm->controls.length () - 1) + { + /* Last iteration: Remain capped to the range [0, VF/N]. */ + gassign *assign = gimple_build_assign (ctrl, MINUS_EXPR, step, + dest_rgm->controls[i - 1]); + gimple_seq_add_stmt (seq, assign); + } + else + { + /* (MIN (remain, VF*I/N)) capped to the range [0, VF/N]. */ + step = gimple_build (seq, MINUS_EXPR, iv_type, step, + dest_rgm->controls[i - 1]); + gassign *assign + = gimple_build_assign (ctrl, MIN_EXPR, step, length_limit); + gimple_seq_add_stmt (seq, assign); + } + } +} + /* Helper for vect_set_loop_condition_partial_vectors. Generate definitions for all the rgroup controls in RGC and return a control that is nonzero when the loop needs to iterate. Add any new preheader statements to @@ -425,7 +482,8 @@ vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo, gimple_seq *header_seq, gimple_stmt_iterator loop_cond_gsi, rgroup_controls *rgc, tree niters, - tree niters_skip, bool might_wrap_p) + tree niters_skip, bool might_wrap_p, + tree *iv_step) { tree compare_type = LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo); tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo); @@ -468,6 +526,39 @@ vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo, gimple_stmt_iterator incr_gsi; bool insert_after; standard_iv_increment_position (loop, &incr_gsi, &insert_after); + if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo)) + { + /* Create an IV that counts down from niters_total and whose step + is the (variable) amount processed in the current iteration: + ... + _10 = (unsigned long) count_12(D); + ... + # ivtmp_9 = PHI <ivtmp_35(6), _10(5)> + _36 = MIN_EXPR <ivtmp_9, POLY_INT_CST [4, 4]>; + ... + vect__4.8_28 = .LEN_LOAD (_17, 32B, _36, 0); + ... + ivtmp_35 = ivtmp_9 - _36; + ... + if (ivtmp_35 != 0) + goto <bb 4>; [83.33%] + else + goto <bb 5>; [16.67%] + */ + nitems_total = gimple_convert (preheader_seq, iv_type, nitems_total); + tree step = rgc->controls.length () == 1 ? rgc->controls[0] + : make_ssa_name (iv_type); + /* Create decrement IV. */ + create_iv (nitems_total, MINUS_EXPR, step, NULL_TREE, loop, &incr_gsi, + insert_after, &index_before_incr, &index_after_incr); + gimple_seq_add_stmt (header_seq, gimple_build_assign (step, MIN_EXPR, + index_before_incr, + nitems_step)); + *iv_step = step; + return index_after_incr; + } + + /* Create increment IV. */ create_iv (build_int_cst (iv_type, 0), PLUS_EXPR, nitems_step, NULL_TREE, loop, &incr_gsi, insert_after, &index_before_incr, &index_after_incr); @@ -733,7 +824,9 @@ vect_set_loop_condition_partial_vectors (class loop *loop, the first control from any rgroup for the loop condition; here we arbitrarily pick the last. */ tree test_ctrl = NULL_TREE; + tree iv_step = NULL_TREE; rgroup_controls *rgc; + rgroup_controls *iv_rgc = nullptr; unsigned int i; auto_vec<rgroup_controls> *controls = use_masks_p ? &LOOP_VINFO_MASKS (loop_vinfo) @@ -753,17 +846,36 @@ vect_set_loop_condition_partial_vectors (class loop *loop, continue; } - /* See whether zero-based IV would ever generate all-false masks - or zero length before wrapping around. */ - bool might_wrap_p = vect_rgroup_iv_might_wrap_p (loop_vinfo, rgc); - - /* Set up all controls for this group. */ - test_ctrl = vect_set_loop_controls_directly (loop, loop_vinfo, - &preheader_seq, - &header_seq, - loop_cond_gsi, rgc, - niters, niters_skip, - might_wrap_p); + if (!LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) + || !iv_rgc + || (iv_rgc->max_nscalars_per_iter * iv_rgc->factor + != rgc->max_nscalars_per_iter * rgc->factor)) + { + /* See whether zero-based IV would ever generate all-false masks + or zero length before wrapping around. */ + bool might_wrap_p = vect_rgroup_iv_might_wrap_p (loop_vinfo, rgc); + + /* Set up all controls for this group. */ + test_ctrl + = vect_set_loop_controls_directly (loop, loop_vinfo, + &preheader_seq, &header_seq, + loop_cond_gsi, rgc, niters, + niters_skip, might_wrap_p, + &iv_step); + + iv_rgc = rgc; + } + + if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) + && rgc->controls.length () > 1) + { + /* vect_set_loop_controls_directly creates an IV whose step + is equal to the expected sum of RGC->controls. Use that + information to populate RGC->controls. */ + tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo); + gcc_assert (iv_step); + vect_adjust_loop_lens_control (iv_type, &header_seq, rgc, iv_step); + } } /* Emit all accumulated statements. */ |