Record gather/scatter scale and base in the SLP tree

The main gather/scatter discovery happens at SLP discovery time,
the base address and the offset scale are currently not explicitly
represented in the SLP tree.  This requires re-discovery of them
during vectorizable_store/load.  The following fixes this by
recording this info into the SLP tree.  This allows the main
vect_check_gather_scatter call to be elided from get_load_store_type
and replaced with target support checks for IFN/decl or fallback
emulated mode.

There's vect_check_gather_scatter left in the path using gather/scatter
for strided load/store.  I hope to deal with this later.

	* tree-vectorizer.h (_slp_tree::gs_scale): New.
	(_slp_tree::gs_base): Likewise.
	(SLP_TREE_GS_SCALE): Likewise.
	(SLP_TREE_GS_BASE): Likewise.
	(vect_describe_gather_scatter_call): Declare.
	* tree-vect-slp.cc (_slp_tree::_slp_tree): Initialize
	new members.
	(vect_build_slp_tree_2): Record gather/scatter base and scale.
	(vect_get_and_check_slp_defs): For gather/scatter IFNs
	describe the call to first_gs_info.
	* tree-vect-data-refs.cc (vect_gather_scatter_fn_p): Add
	mode of operation with fixed offset vector type.
	(vect_describe_gather_scatter_call): Export.
	* tree-vect-stmts.cc (get_load_store_type): Do not call
	vect_check_gather_scatter to fill gs_info, instead populate
	from the SLP tree.  Check which of, IFN, decl or fallback
	is supported and record that decision.

4 files changed, 79 insertions, 27 deletions

diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index dc82bf6..4db0ec8 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -4423,8 +4423,9 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
    MASKED_P is true if the load or store is conditional.  MEMORY_TYPE is
    the type of the memory elements being loaded or stored.  OFFSET_TYPE
    is the type of the offset that is being applied to the invariant
-   base address.  SCALE is the amount by which the offset should
-   be multiplied *after* it has been converted to address width.
+   base address.  If OFFSET_TYPE is scalar the function chooses an
+   appropriate vector type for it.  SCALE is the amount by which the
+   offset should be multiplied *after* it has been converted to address width.
 
    Return true if the function is supported, storing the function id in
    *IFN_OUT and the vector type for the offset in *OFFSET_VECTYPE_OUT.
@@ -4467,9 +4468,15 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p,
 
   for (;;)
     {
-      tree offset_vectype = get_vectype_for_scalar_type (vinfo, offset_type);
-      if (!offset_vectype)
-	return false;
+      tree offset_vectype;
+      if (VECTOR_TYPE_P (offset_type))
+	offset_vectype = offset_type;
+      else
+	{
+	  offset_vectype = get_vectype_for_scalar_type (vinfo, offset_type);
+	  if (!offset_vectype)
+	    return false;
+	}
 
       /* Test whether the target supports this combination.  */
       if (internal_gather_scatter_fn_supported_p (ifn, vectype, memory_type,
@@ -4500,10 +4507,15 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p,
 	  return true;
 	}
 
+      /* For fixed offset vector type we're done.  */
+      if (VECTOR_TYPE_P (offset_type))
+	return false;
+
       if (TYPE_PRECISION (offset_type) >= POINTER_SIZE
 	  && TYPE_PRECISION (offset_type) >= element_bits)
 	return false;
 
+      /* Try a larger offset vector type.  */
       offset_type = build_nonstandard_integer_type
 	(TYPE_PRECISION (offset_type) * 2, TYPE_UNSIGNED (offset_type));
     }
@@ -4512,7 +4524,7 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p,
 /* STMT_INFO is a call to an internal gather load or scatter store function.
    Describe the operation in INFO.  */
 
-static void
+void
 vect_describe_gather_scatter_call (stmt_vec_info stmt_info,
 				   gather_scatter_info *info)
 {
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 794a073..59acbc1 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -120,6 +120,8 @@ _slp_tree::_slp_tree ()
   SLP_TREE_LANE_PERMUTATION (this) = vNULL;
   SLP_TREE_DEF_TYPE (this) = vect_uninitialized_def;
   SLP_TREE_CODE (this) = ERROR_MARK;
+  SLP_TREE_GS_SCALE (this) = 0;
+  SLP_TREE_GS_BASE (this) = NULL_TREE;
   this->ldst_lanes = false;
   this->avoid_stlf_fail = false;
   SLP_TREE_VECTYPE (this) = NULL_TREE;
@@ -680,6 +682,15 @@ vect_get_and_check_slp_defs (vec_info *vinfo, unsigned char swap,
 	{
 	  internal_fn ifn = gimple_call_internal_fn (stmt);
 	  commutative_op = first_commutative_argument (ifn);
+	  if (internal_gather_scatter_fn_p (ifn))
+	    {
+	      vect_describe_gather_scatter_call
+		(stmt_info,
+		 first ? &(*oprnds_info)[0]->first_gs_info : &gs_info);
+	      if (first)
+		(*oprnds_info)[0]->first_gs_p = true;
+	      gs_op = 0;
+	    }
 	}
     }
   else if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
@@ -2720,6 +2731,9 @@ out:
 
   stmt_info = stmts[0];
 
+  int gs_scale = 0;
+  tree gs_base = NULL_TREE;
+
   /* Create SLP_TREE nodes for the definition node/s.  */
   FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
     {
@@ -2742,6 +2756,12 @@ out:
 	  continue;
 	}
 
+      if (oprnd_info->first_gs_p)
+	{
+	  gs_scale = oprnd_info->first_gs_info.scale;
+	  gs_base = oprnd_info->first_gs_info.base;
+	}
+
       if (is_a <bb_vec_info> (vinfo)
 	  && oprnd_info->first_dt == vect_internal_def
 	  && !oprnd_info->any_pattern)
@@ -3131,6 +3151,8 @@ fail:
   node = vect_create_new_slp_node (node, stmts, nops);
   SLP_TREE_VECTYPE (node) = vectype;
   SLP_TREE_CHILDREN (node).splice (children);
+  SLP_TREE_GS_SCALE (node) = gs_scale;
+  SLP_TREE_GS_BASE (node) = gs_base;
   return node;
 }
 
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 3bf2dec..a6f4db4 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -2019,31 +2019,40 @@ get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
   else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
     {
       *memory_access_type = VMAT_GATHER_SCATTER;
-      if (!vect_check_gather_scatter (stmt_info, loop_vinfo, gs_info,
-				      elsvals))
-	gcc_unreachable ();
       slp_tree offset_node = SLP_TREE_CHILDREN (slp_node)[0];
       tree offset_vectype = SLP_TREE_VECTYPE (offset_node);
+      memset (gs_info, 0, sizeof (gather_scatter_info));
       gs_info->offset_vectype = offset_vectype;
-      /* When using internal functions, we rely on pattern recognition
-	 to convert the type of the offset to the type that the target
-	 requires, with the result being a call to an internal function.
-	 If that failed for some reason (e.g. because another pattern
-	 took priority), just handle cases in which the offset already
-	 has the right type.  */
-      if (GATHER_SCATTER_IFN_P (*gs_info)
-	  && !is_gimple_call (stmt_info->stmt)
-	  && !tree_nop_conversion_p (TREE_TYPE (gs_info->offset),
-				     TREE_TYPE (offset_vectype)))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "%s offset requires a conversion\n",
-			     vls_type == VLS_LOAD ? "gather" : "scatter");
-	  return false;
-	}
-      else if (GATHER_SCATTER_EMULATED_P (*gs_info))
+      gs_info->scale = SLP_TREE_GS_SCALE (slp_node);
+      gs_info->base = SLP_TREE_GS_BASE (slp_node);
+      gs_info->memory_type = TREE_TYPE (DR_REF (first_dr_info->dr));
+      gs_info->decl = NULL_TREE;
+      gs_info->ifn = IFN_LAST;
+      tree tem;
+      if (vect_gather_scatter_fn_p (loop_vinfo, vls_type == VLS_LOAD,
+				    masked_p, vectype,
+				    gs_info->memory_type,
+				    offset_vectype, gs_info->scale,
+				    &gs_info->ifn, &tem,
+				    elsvals))
+	/* GATHER_SCATTER_IFN_P.  */;
+      else if (vls_type == VLS_LOAD
+	       ? (targetm.vectorize.builtin_gather
+		  && (gs_info->decl
+			= targetm.vectorize.builtin_gather (vectype,
+							    TREE_TYPE
+							      (offset_vectype),
+							    gs_info->scale)))
+	       : (targetm.vectorize.builtin_scatter
+		  && (gs_info->decl
+			= targetm.vectorize.builtin_scatter (vectype,
+							     TREE_TYPE
+							       (offset_vectype),
+							     gs_info->scale))))
+	/* GATHER_SCATTER_LEGACY_P.  */;
+      else
 	{
+	  /* GATHER_SCATTER_EMULATED_P.  */
 	  if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
 	      || !TYPE_VECTOR_SUBPARTS (offset_vectype).is_constant ()
 	      || VECTOR_BOOLEAN_TYPE_P (offset_vectype)
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 52e1075..9653496 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -306,6 +306,11 @@ struct _slp_tree {
   unsigned int lanes;
   /* The operation of this node.  */
   enum tree_code code;
+  /* For gather/scatter memory operations the scale each offset element
+     should be multiplied by before being added to the base.  */
+  int gs_scale;
+  /* For gather/scatter memory operations the loop-invariant base value.  */
+  tree gs_base;
   /* Whether uses of this load or feeders of this store are suitable
      for load/store-lanes.  */
   bool ldst_lanes;
@@ -412,6 +417,8 @@ public:
 #define SLP_TREE_CODE(S)			 (S)->code
 #define SLP_TREE_MEMORY_ACCESS_TYPE(S)		 (S)->memory_access_type
 #define SLP_TREE_TYPE(S)			 (S)->type
+#define SLP_TREE_GS_SCALE(S)			 (S)->gs_scale
+#define SLP_TREE_GS_BASE(S)			 (S)->gs_base
 
 enum vect_partial_vector_style {
     vect_partial_vectors_none,
@@ -2550,6 +2557,8 @@ extern bool vect_gather_scatter_fn_p (vec_info *, bool, bool, tree, tree,
 extern bool vect_check_gather_scatter (stmt_vec_info, loop_vec_info,
 				       gather_scatter_info *,
 				       vec<int> * = nullptr);
+extern void vect_describe_gather_scatter_call (stmt_vec_info,
+					       gather_scatter_info *);
 extern opt_result vect_find_stmt_data_reference (loop_p, gimple *,
 						 vec<data_reference_p> *,
 						 vec<int> *, int);