Add support for reductions in fully-masked loops

This patch removes the restriction that fully-masked loops cannot
have reductions.  The key thing here is to make sure that the
reduction accumulator doesn't include any values associated with
inactive lanes; the patch adds a bunch of conditional binary
operations for doing that.

2018-01-13  Richard Sandiford  <richard.sandiford@linaro.org>
	    Alan Hayward  <alan.hayward@arm.com>
	    David Sherwood  <david.sherwood@arm.com>

gcc/
	* doc/md.texi (cond_add@var{mode}, cond_sub@var{mode})
	(cond_and@var{mode}, cond_ior@var{mode}, cond_xor@var{mode})
	(cond_smin@var{mode}, cond_smax@var{mode}, cond_umin@var{mode})
	(cond_umax@var{mode}): Document.
	* optabs.def (cond_add_optab, cond_sub_optab, cond_and_optab)
	(cond_ior_optab, cond_xor_optab, cond_smin_optab, cond_smax_optab)
	(cond_umin_optab, cond_umax_optab): New optabs.
	* internal-fn.def (COND_ADD, COND_SUB, COND_MIN, COND_MAX, COND_AND)
	(COND_IOR, COND_XOR): New internal functions.
	* internal-fn.h (get_conditional_internal_fn): Declare.
	* internal-fn.c (cond_binary_direct): New macro.
	(expand_cond_binary_optab_fn): Likewise.
	(direct_cond_binary_optab_supported_p): Likewise.
	(get_conditional_internal_fn): New function.
	* tree-vect-loop.c (vectorizable_reduction): Handle fully-masked loops.
	Cope with reduction statements that are vectorized as calls rather
	than assignments.
	* config/aarch64/aarch64-sve.md (cond_<optab><mode>): New insns.
	* config/aarch64/iterators.md (UNSPEC_COND_ADD, UNSPEC_COND_SUB)
	(UNSPEC_COND_SMAX, UNSPEC_COND_UMAX, UNSPEC_COND_SMIN)
	(UNSPEC_COND_UMIN, UNSPEC_COND_AND, UNSPEC_COND_ORR)
	(UNSPEC_COND_EOR): New unspecs.
	(optab): Add mappings for them.
	(SVE_COND_INT_OP, SVE_COND_FP_OP): New int iterators.
	(sve_int_op, sve_fp_op): New int attributes.

gcc/testsuite/
	* gcc.dg/vect/pr60482.c: Remove XFAIL for variable-length vectors.
	* gcc.target/aarch64/sve/reduc_1.c: Expect the loop operations
	to be predicated.
	* gcc.target/aarch64/sve/slp_5.c: Check for a fully-masked loop.
	* gcc.target/aarch64/sve/slp_7.c: Likewise.
	* gcc.target/aarch64/sve/reduc_5.c: New test.
	* gcc.target/aarch64/sve/slp_13.c: Likewise.
	* gcc.target/aarch64/sve/slp_13_run.c: Likewise.

Co-Authored-By: Alan Hayward <alan.hayward@arm.com>
Co-Authored-By: David Sherwood <david.sherwood@arm.com>

From-SVN: r256626

1 files changed, 36 insertions, 0 deletions

diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c
index 44d5486..bc4027b 100644
--- a/gcc/internal-fn.c
+++ b/gcc/internal-fn.c
@@ -88,6 +88,7 @@ init_internal_fns ()
 #define mask_store_lanes_direct { 0, 0, false }
 #define unary_direct { 0, 0, true }
 #define binary_direct { 0, 0, true }
+#define cond_binary_direct { 1, 1, true }
 #define while_direct { 0, 2, false }
 
 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
@@ -2855,6 +2856,9 @@ expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
   expand_direct_optab_fn (FN, STMT, OPTAB, 2)
 
+#define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
+  expand_direct_optab_fn (FN, STMT, OPTAB, 3)
+
 /* RETURN_TYPE and ARGS are a return type and argument list that are
    in principle compatible with FN (which satisfies direct_internal_fn_p).
    Return the types that should be used to determine whether the
@@ -2928,6 +2932,7 @@ multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
 
 #define direct_unary_optab_supported_p direct_optab_supported_p
 #define direct_binary_optab_supported_p direct_optab_supported_p
+#define direct_cond_binary_optab_supported_p direct_optab_supported_p
 #define direct_mask_load_optab_supported_p direct_optab_supported_p
 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
@@ -3049,6 +3054,37 @@ static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
   0
 };
 
+/* Return a function that performs the conditional form of CODE, i.e.:
+
+     LHS = RHS1 ? RHS2 CODE RHS3 : RHS2
+
+   (operating elementwise if the operands are vectors).  Return IFN_LAST
+   if no such function exists.  */
+
+internal_fn
+get_conditional_internal_fn (tree_code code)
+{
+  switch (code)
+    {
+    case PLUS_EXPR:
+      return IFN_COND_ADD;
+    case MINUS_EXPR:
+      return IFN_COND_SUB;
+    case MIN_EXPR:
+      return IFN_COND_MIN;
+    case MAX_EXPR:
+      return IFN_COND_MAX;
+    case BIT_AND_EXPR:
+      return IFN_COND_AND;
+    case BIT_IOR_EXPR:
+      return IFN_COND_IOR;
+    case BIT_XOR_EXPR:
+      return IFN_COND_XOR;
+    default:
+      return IFN_LAST;
+    }
+}
+
 /* Expand STMT as though it were a call to internal function FN.  */
 
 void