Move negative stride bias out of dr_misalignment

This moves applying of a bias for negative stride accesses out of
dr_misalignment in favor of a more general optional offset argument.
The negative bias is now computed by get_load_store_type and applied
accordingly to determine the alignment support scheme.  Likewise
the peeling/versioning code is adjusted albeit that still assumes
we'll end up with VMAT_CONTIGUOUS_DOWN or VMAT_CONTIGUOUS_REVERSE
but at least when not so (VMAT_STRIDED_SLP is one possibility) then
get_load_store_type will _not_ falsely report an aligned access but
instead an access with known misalignment.

This fixes PR96109.

2021-10-25  Richard Biener  <rguenther@suse.de>

	PR tree-optimization/96109
	* tree-vectorizer.h (dr_misalignment): Add optional offset
	parameter.
	* tree-vect-data-refs.c (dr_misalignment): Likewise.  Remove
	offset applied for negative stride accesses.
	(vect_enhance_data_refs_alignment): Compute negative stride
	access offset and pass it to dr_misalignment.
	* tree-vect-stmts.c (get_negative_load_store_type): Pass
	negative offset to dr_misalignment.
	(get_group_load_store_type): Likewise.
	(get_load_store_type): Likewise.
	(vectorizable_store): Remove asserts about alignment.
	(vectorizable_load): Likewise.

3 files changed, 48 insertions, 33 deletions

diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c
index 46360c5..cda89ef 100644
--- a/gcc/tree-vect-data-refs.c
+++ b/gcc/tree-vect-data-refs.c
@@ -887,10 +887,11 @@ vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance)
   return res;
 }
 
-/* Return the misalignment of DR_INFO accessed in VECTYPE.  */
+/* Return the misalignment of DR_INFO accessed in VECTYPE with OFFSET
+   applied.  */
 
 int
-dr_misalignment (dr_vec_info *dr_info, tree vectype)
+dr_misalignment (dr_vec_info *dr_info, tree vectype, poly_int64 offset)
 {
   HOST_WIDE_INT diff = 0;
   /* Alignment is only analyzed for the first element of a DR group,
@@ -919,13 +920,9 @@ dr_misalignment (dr_vec_info *dr_info, tree vectype)
 		targetm.vectorize.preferred_vector_alignment (vectype)))
     return DR_MISALIGNMENT_UNKNOWN;
 
-  /* If this is a backward running DR then first access in the larger
-     vectype actually is N-1 elements before the address in the DR.
-     Adjust misalign accordingly.  */
-  poly_int64 misalignment = misalign + diff;
-  if (tree_int_cst_sgn (DR_STEP (dr_info->dr)) < 0)
-    misalignment += ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
-		     * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
+  /* Apply the offset from the DR group start and the externally supplied
+     offset which can for example result from a negative stride access.  */
+  poly_int64 misalignment = misalign + diff + offset;
 
   /* vect_compute_data_ref_alignment will have ensured that target_alignment
      is constant and otherwise set misalign to DR_MISALIGNMENT_UNKNOWN.  */
@@ -1549,8 +1546,15 @@ vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
       int misalignment;
       unsigned HOST_WIDE_INT alignment;
 
+      bool negative = tree_int_cst_compare (DR_STEP (dr_info->dr),
+					    size_zero_node) < 0;
+      poly_int64 off = 0;
+      if (negative)
+	off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
+	       * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
+
       if (npeel == 0)
-	misalignment = dr_misalignment (dr_info, vectype);
+	misalignment = dr_misalignment (dr_info, vectype, off);
       else if (dr_info == dr0_info
 	       || vect_dr_aligned_if_peeled_dr_is (dr_info, dr0_info))
 	misalignment = 0;
@@ -1560,7 +1564,7 @@ vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
 	misalignment = DR_MISALIGNMENT_UNKNOWN;
       else
 	{
-	  misalignment = dr_misalignment (dr_info, vectype);
+	  misalignment = dr_misalignment (dr_info, vectype, off);
 	  misalignment += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
 	  misalignment &= alignment - 1;
 	}
@@ -1960,8 +1964,11 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
 	       */
 	      unsigned int target_align =
 		DR_TARGET_ALIGNMENT (dr_info).to_constant ();
-	      unsigned int dr_size = vect_get_scalar_dr_size (dr_info);
-	      unsigned int mis = dr_misalignment (dr_info, vectype);
+	      unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (dr_info);
+	      poly_int64 off = 0;
+	      if (negative)
+		off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
+	      unsigned int mis = dr_misalignment (dr_info, vectype, off);
 	      mis = negative ? mis : -mis;
 	      if (mis != 0)
 		npeel_tmp = (mis & (target_align - 1)) / dr_size;
@@ -2238,8 +2245,13 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
                  updating DR_MISALIGNMENT values.  The peeling factor is the
                  vectorization factor minus the misalignment as an element
                  count.  */
+	      tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+	      poly_int64 off = 0;
+	      if (negative)
+		off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
+		       * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
 	      unsigned int mis
-		= dr_misalignment (dr0_info, STMT_VINFO_VECTYPE (stmt_info));
+		= dr_misalignment (dr0_info, vectype, off);
 	      mis = negative ? mis : -mis;
 	      /* If known_alignment_for_access_p then we have set
 	         DR_MISALIGNMENT which is only done if we know it at compiler
@@ -2383,19 +2395,27 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
       FOR_EACH_VEC_ELT (datarefs, i, dr)
         {
 	  dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
-	  stmt_vec_info stmt_info = dr_info->stmt;
-	  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-	  int misalignment;
-	  if ((misalignment = dr_misalignment (dr_info, vectype)) == 0
-	      || !vect_relevant_for_alignment_p (dr_info))
+	  if (!vect_relevant_for_alignment_p (dr_info))
 	    continue;
 
+	  stmt_vec_info stmt_info = dr_info->stmt;
 	  if (STMT_VINFO_STRIDED_P (stmt_info))
 	    {
 	      do_versioning = false;
 	      break;
 	    }
 
+	  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+	  bool negative = tree_int_cst_compare (DR_STEP (dr),
+						size_zero_node) < 0;
+	  poly_int64 off = 0;
+	  if (negative)
+	    off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
+		   * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
+	  int misalignment;
+	  if ((misalignment = dr_misalignment (dr_info, vectype, off)) == 0)
+	    continue;
+
 	  enum dr_alignment_support supportable_dr_alignment
 	    = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
 					     misalignment);
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index 1f56e10..3cd338a 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -2001,7 +2001,7 @@ get_negative_load_store_type (vec_info *vinfo,
   *poffset = ((-TYPE_VECTOR_SUBPARTS (vectype) + 1)
 	      * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
 
-  int misalignment = dr_misalignment (dr_info, vectype);
+  int misalignment = dr_misalignment (dr_info, vectype, *poffset);
   alignment_support_scheme
     = vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment);
   if (alignment_support_scheme != dr_aligned
@@ -2010,8 +2010,8 @@ get_negative_load_store_type (vec_info *vinfo,
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
 			 "negative step but alignment required.\n");
-      return VMAT_ELEMENTWISE;
       *poffset = 0;
+      return VMAT_ELEMENTWISE;
     }
 
   if (vls_type == VLS_STORE_INVARIANT)
@@ -2320,7 +2320,7 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
     }
   else
     {
-      *misalignment = dr_misalignment (first_dr_info, vectype);
+      *misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
       *alignment_support_scheme
 	= vect_supportable_dr_alignment (vinfo, first_dr_info, vectype,
 					 *misalignment);
@@ -2470,7 +2470,7 @@ get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
 	  else
 	    *memory_access_type = VMAT_CONTIGUOUS;
 	  *misalignment = dr_misalignment (STMT_VINFO_DR_INFO (stmt_info),
-					   vectype);
+					   vectype, *poffset);
 	  *alignment_support_scheme
 	    = vect_supportable_dr_alignment (vinfo,
 					     STMT_VINFO_DR_INFO (stmt_info),
@@ -8252,10 +8252,7 @@ vectorizable_store (vec_info *vinfo,
 
 	      align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
 	      if (alignment_support_scheme == dr_aligned)
-		{
-		  gcc_assert (aligned_access_p (first_dr_info, vectype));
-		  misalign = 0;
-		}
+		misalign = 0;
 	      else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
 		{
 		  align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
@@ -8340,7 +8337,7 @@ vectorizable_store (vec_info *vinfo,
 					  ? dataref_offset
 					  : build_int_cst (ref_type, 0));
 		  if (alignment_support_scheme == dr_aligned)
-		    gcc_assert (aligned_access_p (first_dr_info, vectype));
+		    ;
 		  else
 		    TREE_TYPE (data_ref)
 		      = build_aligned_type (TREE_TYPE (data_ref),
@@ -9589,10 +9586,7 @@ vectorizable_load (vec_info *vinfo,
 		    align =
 		      known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
 		    if (alignment_support_scheme == dr_aligned)
-		      {
-			gcc_assert (aligned_access_p (first_dr_info, vectype));
-			misalign = 0;
-		      }
+		      misalign = 0;
 		    else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
 		      {
 			align = dr_alignment
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 866d813..52cdfbf 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -1606,7 +1606,8 @@ set_dr_misalignment (dr_vec_info *dr_info, int val)
   dr_info->misalignment = val;
 }
 
-extern int dr_misalignment (dr_vec_info *dr_info, tree vectype);
+extern int dr_misalignment (dr_vec_info *dr_info, tree vectype,
+			    poly_int64 offset = 0);
 
 #define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)