diff options
Diffstat (limited to 'gcc/tree-vect-data-refs.c')
| -rw-r--r-- | gcc/tree-vect-data-refs.c | 104 |
1 files changed, 89 insertions, 15 deletions
diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 4e99747..3f37510 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -255,6 +255,15 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, /* Unknown data dependence. */ if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know) { + /* If user asserted safelen consecutive iterations can be + executed concurrently, assume independence. */ + if (loop->safelen >= 2) + { + if (loop->safelen < *max_vf) + *max_vf = loop->safelen; + return false; + } + if (STMT_VINFO_GATHER_P (stmtinfo_a) || STMT_VINFO_GATHER_P (stmtinfo_b)) { @@ -291,6 +300,15 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, /* Known data dependence. */ if (DDR_NUM_DIST_VECTS (ddr) == 0) { + /* If user asserted safelen consecutive iterations can be + executed concurrently, assume independence. */ + if (loop->safelen >= 2) + { + if (loop->safelen < *max_vf) + *max_vf = loop->safelen; + return false; + } + if (STMT_VINFO_GATHER_P (stmtinfo_a) || STMT_VINFO_GATHER_P (stmtinfo_b)) { @@ -2930,6 +2948,7 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo, stmt_vec_info stmt_info; tree base, offset, init; bool gather = false; + bool simd_lane_access = false; int vf; again: @@ -2961,12 +2980,17 @@ again: if (!DR_BASE_ADDRESS (dr) || !DR_OFFSET (dr) || !DR_INIT (dr) || !DR_STEP (dr)) { - /* If target supports vector gather loads, see if they can't - be used. */ - if (loop_vinfo - && DR_IS_READ (dr) + bool maybe_gather + = DR_IS_READ (dr) && !TREE_THIS_VOLATILE (DR_REF (dr)) - && targetm.vectorize.builtin_gather != NULL + && targetm.vectorize.builtin_gather != NULL; + bool maybe_simd_lane_access + = loop_vinfo && loop->simduid; + + /* If target supports vector gather loads, or if this might be + a SIMD lane access, see if they can't be used. */ + if (loop_vinfo + && (maybe_gather || maybe_simd_lane_access) && !nested_in_vect_loop_p (loop, stmt)) { struct data_reference *newdr @@ -2979,14 +3003,59 @@ again: && DR_STEP (newdr) && integer_zerop (DR_STEP (newdr))) { - dr = newdr; - gather = true; + if (maybe_simd_lane_access) + { + tree off = DR_OFFSET (newdr); + STRIP_NOPS (off); + if (TREE_CODE (DR_INIT (newdr)) == INTEGER_CST + && TREE_CODE (off) == MULT_EXPR + && host_integerp (TREE_OPERAND (off, 1), 1)) + { + tree step = TREE_OPERAND (off, 1); + off = TREE_OPERAND (off, 0); + STRIP_NOPS (off); + if (CONVERT_EXPR_P (off) + && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (off, + 0))) + < TYPE_PRECISION (TREE_TYPE (off))) + off = TREE_OPERAND (off, 0); + if (TREE_CODE (off) == SSA_NAME) + { + gimple def = SSA_NAME_DEF_STMT (off); + tree reft = TREE_TYPE (DR_REF (newdr)); + if (gimple_call_internal_p (def) + && gimple_call_internal_fn (def) + == IFN_GOMP_SIMD_LANE) + { + tree arg = gimple_call_arg (def, 0); + gcc_assert (TREE_CODE (arg) == SSA_NAME); + arg = SSA_NAME_VAR (arg); + if (arg == loop->simduid + /* For now. */ + && tree_int_cst_equal + (TYPE_SIZE_UNIT (reft), + step)) + { + DR_OFFSET (newdr) = ssize_int (0); + DR_STEP (newdr) = step; + dr = newdr; + simd_lane_access = true; + } + } + } + } + } + if (!simd_lane_access && maybe_gather) + { + dr = newdr; + gather = true; + } } - else + if (!gather && !simd_lane_access) free_data_ref (newdr); } - if (!gather) + if (!gather && !simd_lane_access) { if (dump_enabled_p ()) { @@ -3013,7 +3082,7 @@ again: if (bb_vinfo) break; - if (gather) + if (gather || simd_lane_access) free_data_ref (dr); return false; } @@ -3046,7 +3115,7 @@ again: if (bb_vinfo) break; - if (gather) + if (gather || simd_lane_access) free_data_ref (dr); return false; } @@ -3065,7 +3134,7 @@ again: if (bb_vinfo) break; - if (gather) + if (gather || simd_lane_access) free_data_ref (dr); return false; } @@ -3086,7 +3155,7 @@ again: if (bb_vinfo) break; - if (gather) + if (gather || simd_lane_access) free_data_ref (dr); return false; } @@ -3221,12 +3290,17 @@ again: if (bb_vinfo) break; - if (gather) + if (gather || simd_lane_access) free_data_ref (dr); return false; } STMT_VINFO_DATA_REF (stmt_info) = dr; + if (simd_lane_access) + { + STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) = true; + datarefs[i] = dr; + } /* Set vectype for STMT. */ scalar_type = TREE_TYPE (DR_REF (dr)); @@ -3247,7 +3321,7 @@ again: if (bb_vinfo) break; - if (gather) + if (gather || simd_lane_access) { STMT_VINFO_DATA_REF (stmt_info) = NULL; free_data_ref (dr); |