diff options
Diffstat (limited to 'gcc')
| -rw-r--r-- | gcc/tree-vect-data-refs.c | 68 | ||||
| -rw-r--r-- | gcc/tree-vectorizer.h | 24 |
2 files changed, 60 insertions, 32 deletions
diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index f43d0f4..967d437 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -887,6 +887,39 @@ vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance) return res; } +/* Return the misalignment of DR_INFO. */ + +int +dr_misalignment (dr_vec_info *dr_info) +{ + if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt)) + { + dr_vec_info *first_dr + = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt)); + int misalign = first_dr->misalignment; + gcc_assert (misalign != DR_MISALIGNMENT_UNINITIALIZED); + if (misalign == DR_MISALIGNMENT_UNKNOWN) + return misalign; + /* vect_analyze_data_ref_accesses guarantees that DR_INIT are + INTEGER_CSTs and the first element in the group has the lowest + address. Likewise vect_compute_data_ref_alignment will + have ensured that target_alignment is constant and otherwise + set misalign to DR_MISALIGNMENT_UNKNOWN. */ + HOST_WIDE_INT diff = (TREE_INT_CST_LOW (DR_INIT (dr_info->dr)) + - TREE_INT_CST_LOW (DR_INIT (first_dr->dr))); + gcc_assert (diff >= 0); + unsigned HOST_WIDE_INT target_alignment_c + = first_dr->target_alignment.to_constant (); + return (misalign + diff) % target_alignment_c; + } + else + { + int misalign = dr_info->misalignment; + gcc_assert (misalign != DR_MISALIGNMENT_UNINITIALIZED); + return misalign; + } +} + /* Record the base alignment guarantee given by DRB, which occurs in STMT_INFO. */ @@ -992,7 +1025,7 @@ vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info) poly_uint64 vector_alignment = exact_div (vect_calculate_target_alignment (dr_info), BITS_PER_UNIT); - DR_TARGET_ALIGNMENT (dr_info) = vector_alignment; + SET_DR_TARGET_ALIGNMENT (dr_info, vector_alignment); /* If the main loop has peeled for alignment we have no way of knowing whether the data accesses in the epilogues are aligned. We can't at @@ -2408,7 +2441,12 @@ vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo) { dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr); if (STMT_VINFO_VECTORIZABLE (dr_info->stmt)) - vect_compute_data_ref_alignment (loop_vinfo, dr_info); + { + if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt) + && DR_GROUP_FIRST_ELEMENT (dr_info->stmt) != dr_info->stmt) + continue; + vect_compute_data_ref_alignment (loop_vinfo, dr_info); + } } return opt_result::success (); @@ -2420,13 +2458,9 @@ vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo) static bool vect_slp_analyze_node_alignment (vec_info *vinfo, slp_tree node) { - /* We vectorize from the first scalar stmt in the node unless - the node is permuted in which case we start from the first - element in the group. */ + /* Alignment is maintained in the first element of the group. */ stmt_vec_info first_stmt_info = SLP_TREE_SCALAR_STMTS (node)[0]; - dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info); - if (SLP_TREE_LOAD_PERMUTATION (node).exists ()) - first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info); + first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info); /* We need to commit to a vector type for the group now. */ if (is_a <bb_vec_info> (vinfo) @@ -2440,22 +2474,8 @@ vect_slp_analyze_node_alignment (vec_info *vinfo, slp_tree node) } dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_stmt_info); - vect_compute_data_ref_alignment (vinfo, dr_info); - /* In several places we need alignment of the first element anyway. */ - if (dr_info != first_dr_info) - vect_compute_data_ref_alignment (vinfo, first_dr_info); - - /* For creating the data-ref pointer we need alignment of the - first element as well. */ - first_stmt_info - = vect_stmt_to_vectorize (vect_find_first_scalar_stmt_in_slp (node)); - if (first_stmt_info != SLP_TREE_SCALAR_STMTS (node)[0]) - { - first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info); - if (dr_info != first_dr_info) - vect_compute_data_ref_alignment (vinfo, first_dr_info); - } - + if (dr_info->misalignment == DR_MISALIGNMENT_UNINITIALIZED) + vect_compute_data_ref_alignment (vinfo, dr_info); return true; } diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 7453d2a..c4c5678 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -1606,13 +1606,7 @@ set_dr_misalignment (dr_vec_info *dr_info, int val) dr_info->misalignment = val; } -inline int -dr_misalignment (dr_vec_info *dr_info) -{ - int misalign = dr_info->misalignment; - gcc_assert (misalign != DR_MISALIGNMENT_UNINITIALIZED); - return misalign; -} +extern int dr_misalignment (dr_vec_info *dr_info); /* Reflects actual alignment of first access in the vectorized loop, taking into account peeling/versioning if applied. */ @@ -1620,7 +1614,21 @@ dr_misalignment (dr_vec_info *dr_info) #define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL) /* Only defined once DR_MISALIGNMENT is defined. */ -#define DR_TARGET_ALIGNMENT(DR) ((DR)->target_alignment) +static inline const poly_uint64 +dr_target_alignment (dr_vec_info *dr_info) +{ + if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt)) + dr_info = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt)); + return dr_info->target_alignment; +} +#define DR_TARGET_ALIGNMENT(DR) dr_target_alignment (DR) + +static inline void +set_dr_target_alignment (dr_vec_info *dr_info, poly_uint64 val) +{ + dr_info->target_alignment = val; +} +#define SET_DR_TARGET_ALIGNMENT(DR, VAL) set_dr_target_alignment (DR, VAL) /* Return true if data access DR_INFO is aligned to its target alignment (which may be less than a full vector). */ |