vect: inbranch SIMD clones

There has been support for generating "inbranch" SIMD clones for a long time,
but nothing actually uses them (as far as I can see).

This patch add supports for a sub-set of possible cases (those using
mask_mode == VOIDmode).  The other cases fail to vectorize, just as before,
so there should be no regressions.

The sub-set of support should cover all cases needed by amdgcn, at present.

gcc/ChangeLog:

	* internal-fn.cc (expand_MASK_CALL): New.
	* internal-fn.def (MASK_CALL): New.
	* internal-fn.h (expand_MASK_CALL): New prototype.
	* omp-simd-clone.cc (simd_clone_adjust_argument_types): Set vector_type
	for mask arguments also.
	* tree-if-conv.cc: Include cgraph.h.
	(if_convertible_stmt_p): Do if conversions for calls to SIMD calls.
	(predicate_statements): Convert functions to IFN_MASK_CALL.
	* tree-vect-loop.cc (vect_get_datarefs_in_loop): Recognise
	IFN_MASK_CALL as a SIMD function call.
	* tree-vect-stmts.cc (vectorizable_simd_clone_call): Handle
	IFN_MASK_CALL as an inbranch SIMD function call.
	Generate the mask vector arguments.

gcc/testsuite/ChangeLog:

	* gcc.dg/vect/vect-simd-clone-16.c: New test.
	* gcc.dg/vect/vect-simd-clone-16b.c: New test.
	* gcc.dg/vect/vect-simd-clone-16c.c: New test.
	* gcc.dg/vect/vect-simd-clone-16d.c: New test.
	* gcc.dg/vect/vect-simd-clone-16e.c: New test.
	* gcc.dg/vect/vect-simd-clone-16f.c: New test.
	* gcc.dg/vect/vect-simd-clone-17.c: New test.
	* gcc.dg/vect/vect-simd-clone-17b.c: New test.
	* gcc.dg/vect/vect-simd-clone-17c.c: New test.
	* gcc.dg/vect/vect-simd-clone-17d.c: New test.
	* gcc.dg/vect/vect-simd-clone-17e.c: New test.
	* gcc.dg/vect/vect-simd-clone-17f.c: New test.
	* gcc.dg/vect/vect-simd-clone-18.c: New test.
	* gcc.dg/vect/vect-simd-clone-18b.c: New test.
	* gcc.dg/vect/vect-simd-clone-18c.c: New test.
	* gcc.dg/vect/vect-simd-clone-18d.c: New test.
	* gcc.dg/vect/vect-simd-clone-18e.c: New test.
	* gcc.dg/vect/vect-simd-clone-18f.c: New test.

1 files changed, 38 insertions, 3 deletions

diff --git a/gcc/tree-if-conv.cc b/gcc/tree-if-conv.cc
index b19eebd..a7a8406 100644
--- a/gcc/tree-if-conv.cc
+++ b/gcc/tree-if-conv.cc
@@ -123,6 +123,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-ssa-dse.h"
 #include "tree-vectorizer.h"
 #include "tree-eh.h"
+#include "cgraph.h"
 
 /* For lang_hooks.types.type_for_mode.  */
 #include "langhooks.h"
@@ -1063,7 +1064,8 @@ if_convertible_gimple_assign_stmt_p (gimple *stmt,
    A statement is if-convertible if:
    - it is an if-convertible GIMPLE_ASSIGN,
    - it is a GIMPLE_LABEL or a GIMPLE_COND,
-   - it is builtins call.  */
+   - it is builtins call,
+   - it is a call to a function with a SIMD clone.  */
 
 static bool
 if_convertible_stmt_p (gimple *stmt, vec<data_reference_p> refs)
@@ -1083,13 +1085,23 @@ if_convertible_stmt_p (gimple *stmt, vec<data_reference_p> refs)
 	tree fndecl = gimple_call_fndecl (stmt);
 	if (fndecl)
 	  {
+	    /* We can vectorize some builtins and functions with SIMD
+	       "inbranch" clones.  */
 	    int flags = gimple_call_flags (stmt);
+	    struct cgraph_node *node = cgraph_node::get (fndecl);
 	    if ((flags & ECF_CONST)
 		&& !(flags & ECF_LOOPING_CONST_OR_PURE)
-		/* We can only vectorize some builtins at the moment,
-		   so restrict if-conversion to those.  */
 		&& fndecl_built_in_p (fndecl))
 	      return true;
+	    if (node && node->simd_clones != NULL)
+	      /* Ensure that at least one clone can be "inbranch".  */
+	      for (struct cgraph_node *n = node->simd_clones; n != NULL;
+		   n = n->simdclone->next_clone)
+		if (n->simdclone->inbranch)
+		  {
+		    need_to_predicate = true;
+		    return true;
+		  }
 	  }
 	return false;
       }
@@ -2613,6 +2625,29 @@ predicate_statements (loop_p loop)
 	      gimple_assign_set_rhs1 (stmt, ifc_temp_var (type, rhs, &gsi));
 	      update_stmt (stmt);
 	    }
+
+	  /* Convert functions that have a SIMD clone to IFN_MASK_CALL.  This
+	     will cause the vectorizer to match the "in branch" clone variants,
+	     and serves to build the mask vector in a natural way.  */
+	  gcall *call = dyn_cast <gcall *> (gsi_stmt (gsi));
+	  if (call && !gimple_call_internal_p (call))
+	    {
+	      tree orig_fn = gimple_call_fn (call);
+	      int orig_nargs = gimple_call_num_args (call);
+	      auto_vec<tree> args;
+	      args.safe_push (orig_fn);
+	      for (int i = 0; i < orig_nargs; i++)
+		args.safe_push (gimple_call_arg (call, i));
+	      args.safe_push (cond);
+
+	      /* Replace the call with a IFN_MASK_CALL that has the extra
+		 condition parameter. */
+	      gcall *new_call = gimple_build_call_internal_vec (IFN_MASK_CALL,
+								args);
+	      gimple_call_set_lhs (new_call, gimple_call_lhs (call));
+	      gsi_replace (&gsi, new_call, true);
+	    }
+
 	  lhs = gimple_get_lhs (gsi_stmt (gsi));
 	  if (lhs && TREE_CODE (lhs) == SSA_NAME)
 	    ssa_names.add (lhs);