tm.texi.in (TARGET_VECTORIZE_BUILTIN_SCATTER): New.

gcc/
	* doc/tm.texi.in (TARGET_VECTORIZE_BUILTIN_SCATTER): New.
	* doc/tm.texi: Regenerate.
	* target.def: Add scatter builtin.
	* tree-vectorizer.h: Rename gather_p to gather_scatter_p and use it
	for loads/stores in case of gather/scatter accordingly.
	(STMT_VINFO_GATHER_SCATTER_P(S)): Use it instead of STMT_VINFO_GATHER_P(S).
	(vect_check_gather): Rename to ...
	(vect_check_gather_scatter): this.
	* tree-vect-data-refs.c (vect_analyze_data_ref_dependence): Use
	STMT_VINFO_GATHER_SCATTER_P instead of STMT_VINFO_SCATTER_P.
	(vect_check_gather_scatter): Use it instead of vect_check_gather.
	(vect_analyze_data_refs): Add gatherscatter enum and maybe_scatter variable
	and new checkings for it accordingly.
	* tree-vect-stmts.c
	(STMT_VINFO_GATHER_SCATTER_P(S)): Use it instead of STMT_VINFO_GATHER_P(S).
	(vect_check_gather_scatter): Use it instead of vect_check_gather.
	(vectorizable_store): Add checkings for STMT_VINFO_GATHER_SCATTER_P.

Co-Authored-By: Kirill Yukhin <kirill.yukhin@intel.com>
Co-Authored-By: Petr Murzin <petr.murzin@intel.com>

From-SVN: r227481

1 files changed, 51 insertions, 42 deletions

diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c
index f1eaef4..2439bd6 100644
--- a/gcc/tree-vect-data-refs.c
+++ b/gcc/tree-vect-data-refs.c
@@ -267,8 +267,8 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
 	  return false;
 	}
 
-      if (STMT_VINFO_GATHER_P (stmtinfo_a)
-	  || STMT_VINFO_GATHER_P (stmtinfo_b))
+      if (STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a)
+	  || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b))
 	{
 	  if (dump_enabled_p ())
 	    {
@@ -315,8 +315,8 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
 	  return false;
 	}
 
-      if (STMT_VINFO_GATHER_P (stmtinfo_a)
-	  || STMT_VINFO_GATHER_P (stmtinfo_b))
+      if (STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a)
+	  || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b))
 	{
 	  if (dump_enabled_p ())
 	    {
@@ -2344,10 +2344,7 @@ vect_analyze_data_ref_access (struct data_reference *dr)
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_NOTE, vect_location,
 	                     "zero step in outer loop.\n");
-	  if (DR_IS_READ (dr))
-  	    return true;
-	  else
-	    return false;
+	  return DR_IS_READ (dr);
 	}
     }
 
@@ -2997,12 +2994,12 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
   return true;
 }
 
-/* Check whether a non-affine read in stmt is suitable for gather load
-   and if so, return a builtin decl for that operation.  */
+/* Check whether a non-affine read or write in stmt is suitable for gather load
+   or scatter store and if so, return a builtin decl for that operation.  */
 
 tree
-vect_check_gather (gimple stmt, loop_vec_info loop_vinfo, tree *basep,
-		   tree *offp, int *scalep)
+vect_check_gather_scatter (gimple stmt, loop_vec_info loop_vinfo, tree *basep,
+			   tree *offp, int *scalep)
 {
   HOST_WIDE_INT scale = 1, pbitpos, pbitsize;
   struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
@@ -3031,7 +3028,7 @@ vect_check_gather (gimple stmt, loop_vec_info loop_vinfo, tree *basep,
 	base = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
     }
 
-  /* The gather builtins need address of the form
+  /* The gather and scatter builtins need address of the form
      loop_invariant + vector * {1, 2, 4, 8}
      or
      loop_invariant + sign_extend (vector) * { 1, 2, 4, 8 }.
@@ -3194,8 +3191,13 @@ vect_check_gather (gimple stmt, loop_vec_info loop_vinfo, tree *basep,
   if (offtype == NULL_TREE)
     offtype = TREE_TYPE (off);
 
-  decl = targetm.vectorize.builtin_gather (STMT_VINFO_VECTYPE (stmt_info),
-					   offtype, scale);
+  if (DR_IS_READ (dr))
+    decl = targetm.vectorize.builtin_gather (STMT_VINFO_VECTYPE (stmt_info),
+					     offtype, scale);
+  else
+    decl = targetm.vectorize.builtin_scatter (STMT_VINFO_VECTYPE (stmt_info),
+					      offtype, scale);
+
   if (decl == NULL_TREE)
     return NULL_TREE;
 
@@ -3344,7 +3346,7 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo,
       gimple stmt;
       stmt_vec_info stmt_info;
       tree base, offset, init;
-      bool gather = false;
+      enum { SG_NONE, GATHER, SCATTER } gatherscatter = SG_NONE;
       bool simd_lane_access = false;
       int vf;
 
@@ -3383,18 +3385,22 @@ again:
 	    = DR_IS_READ (dr)
 	      && !TREE_THIS_VOLATILE (DR_REF (dr))
 	      && targetm.vectorize.builtin_gather != NULL;
+	  bool maybe_scatter
+	    = DR_IS_WRITE (dr)
+	      && !TREE_THIS_VOLATILE (DR_REF (dr))
+	      && targetm.vectorize.builtin_scatter != NULL;
 	  bool maybe_simd_lane_access
 	    = loop_vinfo && loop->simduid;
 
-	  /* If target supports vector gather loads, or if this might be
-	     a SIMD lane access, see if they can't be used.  */
+	  /* If target supports vector gather loads or scatter stores, or if
+	     this might be a SIMD lane access, see if they can't be used.  */
 	  if (loop_vinfo
-	      && (maybe_gather || maybe_simd_lane_access)
+	      && (maybe_gather || maybe_scatter || maybe_simd_lane_access)
 	      && !nested_in_vect_loop_p (loop, stmt))
 	    {
 	      struct data_reference *newdr
 		= create_data_ref (NULL, loop_containing_stmt (stmt),
-				   DR_REF (dr), stmt, true);
+				   DR_REF (dr), stmt, maybe_scatter ? false : true);
 	      gcc_assert (newdr != NULL && DR_REF (newdr));
 	      if (DR_BASE_ADDRESS (newdr)
 		  && DR_OFFSET (newdr)
@@ -3447,17 +3453,20 @@ again:
 			    }
 			}
 		    }
-		  if (!simd_lane_access && maybe_gather)
+		  if (!simd_lane_access && (maybe_gather || maybe_scatter))
 		    {
 		      dr = newdr;
-		      gather = true;
+		      if (maybe_gather)
+			gatherscatter = GATHER;
+		      else
+			gatherscatter = SCATTER;
 		    }
 		}
-	      if (!gather && !simd_lane_access)
+	      if (gatherscatter == SG_NONE && !simd_lane_access)
 		free_data_ref (newdr);
 	    }
 
-	  if (!gather && !simd_lane_access)
+	  if (gatherscatter == SG_NONE && !simd_lane_access)
 	    {
 	      if (dump_enabled_p ())
 		{
@@ -3485,7 +3494,7 @@ again:
           if (bb_vinfo)
 	    break;
 
-	  if (gather || simd_lane_access)
+	  if (gatherscatter != SG_NONE || simd_lane_access)
 	    free_data_ref (dr);
 	  return false;
         }
@@ -3520,7 +3529,7 @@ again:
           if (bb_vinfo)
 	    break;
 
-	  if (gather || simd_lane_access)
+	  if (gatherscatter != SG_NONE || simd_lane_access)
 	    free_data_ref (dr);
           return false;
         }
@@ -3540,7 +3549,7 @@ again:
           if (bb_vinfo)
 	    break;
 
-	  if (gather || simd_lane_access)
+	  if (gatherscatter != SG_NONE || simd_lane_access)
 	    free_data_ref (dr);
           return false;
 	}
@@ -3565,7 +3574,7 @@ again:
 	  if (bb_vinfo)
 	    break;
 
-	  if (gather || simd_lane_access)
+	  if (gatherscatter != SG_NONE || simd_lane_access)
 	    free_data_ref (dr);
 	  return false;
 	}
@@ -3703,7 +3712,7 @@ again:
           if (bb_vinfo)
 	    break;
 
-	  if (gather || simd_lane_access)
+	  if (gatherscatter != SG_NONE || simd_lane_access)
 	    free_data_ref (dr);
           return false;
         }
@@ -3736,10 +3745,10 @@ again:
           if (bb_vinfo)
 	    break;
 
-	  if (gather || simd_lane_access)
+	  if (gatherscatter != SG_NONE || simd_lane_access)
 	    {
 	      STMT_VINFO_DATA_REF (stmt_info) = NULL;
-	      if (gather)
+	      if (gatherscatter != SG_NONE)
 		free_data_ref (dr);
 	    }
 	  return false;
@@ -3763,23 +3772,22 @@ again:
       if (vf > *min_vf)
 	*min_vf = vf;
 
-      if (gather)
+      if (gatherscatter != SG_NONE)
 	{
 	  tree off;
-
-	  gather = 0 != vect_check_gather (stmt, loop_vinfo, NULL, &off, NULL);
-	  if (gather
-	      && get_vectype_for_scalar_type (TREE_TYPE (off)) == NULL_TREE)
-	    gather = false;
-	  if (!gather)
+	  if (!vect_check_gather_scatter (stmt, loop_vinfo, NULL, &off, NULL)
+	      || get_vectype_for_scalar_type (TREE_TYPE (off)) == NULL_TREE)
 	    {
 	      STMT_VINFO_DATA_REF (stmt_info) = NULL;
 	      free_data_ref (dr);
 	      if (dump_enabled_p ())
 		{
-		  dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, 
-                                   "not vectorized: not suitable for gather "
-                                   "load ");
+		  dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+				   (gatherscatter == GATHER) ?
+				   "not vectorized: not suitable for gather "
+				   "load " :
+				   "not vectorized: not suitable for scatter "
+				   "store ");
 		  dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
                   dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
 		}
@@ -3787,8 +3795,9 @@ again:
 	    }
 
 	  datarefs[i] = dr;
-	  STMT_VINFO_GATHER_P (stmt_info) = true;
+	  STMT_VINFO_GATHER_SCATTER_P (stmt_info) = gatherscatter;
 	}
+
       else if (loop_vinfo
 	       && TREE_CODE (DR_STEP (dr)) != INTEGER_CST)
 	{