Add support for SVE scatter stores

This is mostly a mechanical extension of the previous gather load
support to scatter stores.  The internal functions in this case are:

  IFN_SCATTER_STORE (base, offsets, scale, values)
  IFN_MASK_SCATTER_STORE (base, offsets, scale, values, mask)

However, one nonobvious change is to vect_analyze_data_ref_access.
If we're treating an access as a gather load or scatter store
(i.e. if STMT_VINFO_GATHER_SCATTER_P is true), the existing code
would create a dummy data_reference whose step is 0.  There's not
really much else it could do, since the whole point is that the
step isn't predictable from iteration to iteration.  We then
went into this code in vect_analyze_data_ref_access:

  /* Allow loads with zero step in inner-loop vectorization.  */
  if (loop_vinfo && integer_zerop (step))
    {
      GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = NULL;
      if (!nested_in_vect_loop_p (loop, stmt))
	return DR_IS_READ (dr);

I.e. we'd take the step literally and assume that this is a load
or store to an invariant address.  Loads from invariant addresses
are supported but stores to them aren't.

The code therefore had the effect of disabling all scatter stores.
AFAICT this is true of AVX too: although tests like avx512f-scatter-1.c
test for the correctness of a scatter-like loop, they don't seem to
check whether a scatter instruction is actually used.

The patch therefore makes vect_analyze_data_ref_access return true
for scatters.  We do seem to handle the aliasing correctly;
that's tested by other functions, and is symmetrical to the
already-working gather case.

2018-01-13  Richard Sandiford  <richard.sandiford@linaro.org>
	    Alan Hayward  <alan.hayward@arm.com>
	    David Sherwood  <david.sherwood@arm.com>

gcc/
	* doc/sourcebuild.texi (vect_scatter_store): Document.
	* optabs.def (scatter_store_optab, mask_scatter_store_optab): New
	optabs.
	* doc/md.texi (scatter_store@var{m}, mask_scatter_store@var{m}):
	Document.
	* genopinit.c (main): Add supports_vec_scatter_store and
	supports_vec_scatter_store_cached to target_optabs.
	* gimple.h (gimple_expr_type): Handle IFN_SCATTER_STORE and
	IFN_MASK_SCATTER_STORE.
	* internal-fn.def (SCATTER_STORE, MASK_SCATTER_STORE): New internal
	functions.
	* internal-fn.h (internal_store_fn_p): Declare.
	(internal_fn_stored_value_index): Likewise.
	* internal-fn.c (scatter_store_direct): New macro.
	(expand_scatter_store_optab_fn): New function.
	(direct_scatter_store_optab_supported_p): New macro.
	(internal_store_fn_p): New function.
	(internal_gather_scatter_fn_p): Handle IFN_SCATTER_STORE and
	IFN_MASK_SCATTER_STORE.
	(internal_fn_mask_index): Likewise.
	(internal_fn_stored_value_index): New function.
	(internal_gather_scatter_fn_supported_p): Adjust operand numbers
	for scatter stores.
	* optabs-query.h (supports_vec_scatter_store_p): Declare.
	* optabs-query.c (supports_vec_scatter_store_p): New function.
	* tree-vectorizer.h (vect_get_store_rhs): Declare.
	* tree-vect-data-refs.c (vect_analyze_data_ref_access): Return
	true for scatter stores.
	(vect_gather_scatter_fn_p): Handle scatter stores too.
	(vect_check_gather_scatter): Consider using scatter stores if
	supports_vec_scatter_store_p.
	* tree-vect-patterns.c (vect_try_gather_scatter_pattern): Handle
	scatter stores too.
	* tree-vect-stmts.c (exist_non_indexing_operands_for_use_p): Use
	internal_fn_stored_value_index.
	(check_load_store_masking): Handle scatter stores too.
	(vect_get_store_rhs): Make public.
	(vectorizable_call): Use internal_store_fn_p.
	(vectorizable_store): Handle scatter store internal functions.
	(vect_transform_stmt): Compare GROUP_STORE_COUNT with GROUP_SIZE
	when deciding whether the end of the group has been reached.
	* config/aarch64/aarch64.md (UNSPEC_ST1_SCATTER): New unspec.
	* config/aarch64/aarch64-sve.md (scatter_store<mode>): New expander.
	(mask_scatter_store<mode>): New insns.

gcc/testsuite/
	* lib/target-supports.exp (check_effective_target_vect_scatter_store):
	New proc.
	* gcc.dg/vect/pr25413a.c: Expect both loops to be optimized on
	targets with scatter stores.
	* gcc.dg/vect/vect-71.c: Restrict XFAIL to targets without scatter
	stores.
	* gcc.target/aarch64/sve/mask_scatter_store_1.c: New test.
	* gcc.target/aarch64/sve/mask_scatter_store_2.c: Likewise.
	* gcc.target/aarch64/sve/scatter_store_1.c: Likewise.
	* gcc.target/aarch64/sve/scatter_store_2.c: Likewise.
	* gcc.target/aarch64/sve/scatter_store_3.c: Likewise.
	* gcc.target/aarch64/sve/scatter_store_4.c: Likewise.
	* gcc.target/aarch64/sve/scatter_store_5.c: Likewise.
	* gcc.target/aarch64/sve/scatter_store_6.c: Likewise.
	* gcc.target/aarch64/sve/scatter_store_7.c: Likewise.
	* gcc.target/aarch64/sve/strided_store_1.c: Likewise.
	* gcc.target/aarch64/sve/strided_store_2.c: Likewise.
	* gcc.target/aarch64/sve/strided_store_3.c: Likewise.
	* gcc.target/aarch64/sve/strided_store_4.c: Likewise.
	* gcc.target/aarch64/sve/strided_store_5.c: Likewise.
	* gcc.target/aarch64/sve/strided_store_6.c: Likewise.
	* gcc.target/aarch64/sve/strided_store_7.c: Likewise.

Co-Authored-By: Alan Hayward <alan.hayward@arm.com>
Co-Authored-By: David Sherwood <david.sherwood@arm.com>

From-SVN: r256643

1 files changed, 85 insertions, 2 deletions

diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c
index 8cf5b79..88adaea 100644
--- a/gcc/internal-fn.c
+++ b/gcc/internal-fn.c
@@ -87,6 +87,7 @@ init_internal_fns ()
 #define mask_store_direct { 3, 2, false }
 #define store_lanes_direct { 0, 0, false }
 #define mask_store_lanes_direct { 0, 0, false }
+#define scatter_store_direct { 3, 3, false }
 #define unary_direct { 0, 0, true }
 #define binary_direct { 0, 0, true }
 #define cond_unary_direct { 1, 1, true }
@@ -2730,6 +2731,42 @@ expand_LAUNDER (internal_fn, gcall *call)
   expand_assignment (lhs, gimple_call_arg (call, 0), false);
 }
 
+/* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB.  */
+
+static void
+expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
+{
+  internal_fn ifn = gimple_call_internal_fn (stmt);
+  int rhs_index = internal_fn_stored_value_index (ifn);
+  int mask_index = internal_fn_mask_index (ifn);
+  tree base = gimple_call_arg (stmt, 0);
+  tree offset = gimple_call_arg (stmt, 1);
+  tree scale = gimple_call_arg (stmt, 2);
+  tree rhs = gimple_call_arg (stmt, rhs_index);
+
+  rtx base_rtx = expand_normal (base);
+  rtx offset_rtx = expand_normal (offset);
+  HOST_WIDE_INT scale_int = tree_to_shwi (scale);
+  rtx rhs_rtx = expand_normal (rhs);
+
+  struct expand_operand ops[6];
+  int i = 0;
+  create_address_operand (&ops[i++], base_rtx);
+  create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
+  create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
+  create_integer_operand (&ops[i++], scale_int);
+  create_input_operand (&ops[i++], rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
+  if (mask_index >= 0)
+    {
+      tree mask = gimple_call_arg (stmt, mask_index);
+      rtx mask_rtx = expand_normal (mask);
+      create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
+    }
+
+  insn_code icode = direct_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)));
+  expand_insn (icode, i, ops);
+}
+
 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB.  */
 
 static void
@@ -3016,6 +3053,7 @@ multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
 #define direct_mask_store_optab_supported_p direct_optab_supported_p
 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
+#define direct_scatter_store_optab_supported_p direct_optab_supported_p
 #define direct_while_optab_supported_p convert_optab_supported_p
 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
 #define direct_fold_left_optab_supported_p direct_optab_supported_p
@@ -3202,6 +3240,25 @@ internal_load_fn_p (internal_fn fn)
     }
 }
 
+/* Return true if IFN is some form of store to memory.  */
+
+bool
+internal_store_fn_p (internal_fn fn)
+{
+  switch (fn)
+    {
+    case IFN_MASK_STORE:
+    case IFN_STORE_LANES:
+    case IFN_MASK_STORE_LANES:
+    case IFN_SCATTER_STORE:
+    case IFN_MASK_SCATTER_STORE:
+      return true;
+
+    default:
+      return false;
+    }
+}
+
 /* Return true if IFN is some form of gather load or scatter store.  */
 
 bool
@@ -3211,6 +3268,8 @@ internal_gather_scatter_fn_p (internal_fn fn)
     {
     case IFN_GATHER_LOAD:
     case IFN_MASK_GATHER_LOAD:
+    case IFN_SCATTER_STORE:
+    case IFN_MASK_SCATTER_STORE:
       return true;
 
     default:
@@ -3235,6 +3294,27 @@ internal_fn_mask_index (internal_fn fn)
     case IFN_MASK_GATHER_LOAD:
       return 3;
 
+    case IFN_MASK_SCATTER_STORE:
+      return 4;
+
+    default:
+      return -1;
+    }
+}
+
+/* If FN takes a value that should be stored to memory, return the index
+   of that argument, otherwise return -1.  */
+
+int
+internal_fn_stored_value_index (internal_fn fn)
+{
+  switch (fn)
+    {
+    case IFN_MASK_STORE:
+    case IFN_SCATTER_STORE:
+    case IFN_MASK_SCATTER_STORE:
+      return 3;
+
     default:
       return -1;
     }
@@ -3259,9 +3339,12 @@ internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
     return false;
   optab optab = direct_internal_fn_optab (ifn);
   insn_code icode = direct_optab_handler (optab, TYPE_MODE (vector_type));
+  int output_ops = internal_load_fn_p (ifn) ? 1 : 0;
   return (icode != CODE_FOR_nothing
-	  && insn_operand_matches (icode, 3, GEN_INT (offset_sign == UNSIGNED))
-	  && insn_operand_matches (icode, 4, GEN_INT (scale)));
+	  && insn_operand_matches (icode, 2 + output_ops,
+				   GEN_INT (offset_sign == UNSIGNED))
+	  && insn_operand_matches (icode, 3 + output_ops,
+				   GEN_INT (scale)));
 }
 
 /* Expand STMT as though it were a call to internal function FN.  */