From 79e4f386ad836b7f9e1bdaf6aded4585035432ef Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Mon, 4 Aug 2025 11:27:54 +0200 Subject: Record gather/scatter scale and base in the SLP tree The main gather/scatter discovery happens at SLP discovery time, the base address and the offset scale are currently not explicitly represented in the SLP tree. This requires re-discovery of them during vectorizable_store/load. The following fixes this by recording this info into the SLP tree. This allows the main vect_check_gather_scatter call to be elided from get_load_store_type and replaced with target support checks for IFN/decl or fallback emulated mode. There's vect_check_gather_scatter left in the path using gather/scatter for strided load/store. I hope to deal with this later. * tree-vectorizer.h (_slp_tree::gs_scale): New. (_slp_tree::gs_base): Likewise. (SLP_TREE_GS_SCALE): Likewise. (SLP_TREE_GS_BASE): Likewise. (vect_describe_gather_scatter_call): Declare. * tree-vect-slp.cc (_slp_tree::_slp_tree): Initialize new members. (vect_build_slp_tree_2): Record gather/scatter base and scale. (vect_get_and_check_slp_defs): For gather/scatter IFNs describe the call to first_gs_info. * tree-vect-data-refs.cc (vect_gather_scatter_fn_p): Add mode of operation with fixed offset vector type. (vect_describe_gather_scatter_call): Export. * tree-vect-stmts.cc (get_load_store_type): Do not call vect_check_gather_scatter to fill gs_info, instead populate from the SLP tree. Check which of, IFN, decl or fallback is supported and record that decision. --- gcc/tree-vectorizer.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'gcc/tree-vectorizer.h') diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 52e1075..9653496 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -306,6 +306,11 @@ struct _slp_tree { unsigned int lanes; /* The operation of this node. */ enum tree_code code; + /* For gather/scatter memory operations the scale each offset element + should be multiplied by before being added to the base. */ + int gs_scale; + /* For gather/scatter memory operations the loop-invariant base value. */ + tree gs_base; /* Whether uses of this load or feeders of this store are suitable for load/store-lanes. */ bool ldst_lanes; @@ -412,6 +417,8 @@ public: #define SLP_TREE_CODE(S) (S)->code #define SLP_TREE_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type #define SLP_TREE_TYPE(S) (S)->type +#define SLP_TREE_GS_SCALE(S) (S)->gs_scale +#define SLP_TREE_GS_BASE(S) (S)->gs_base enum vect_partial_vector_style { vect_partial_vectors_none, @@ -2550,6 +2557,8 @@ extern bool vect_gather_scatter_fn_p (vec_info *, bool, bool, tree, tree, extern bool vect_check_gather_scatter (stmt_vec_info, loop_vec_info, gather_scatter_info *, vec * = nullptr); +extern void vect_describe_gather_scatter_call (stmt_vec_info, + gather_scatter_info *); extern opt_result vect_find_stmt_data_reference (loop_p, gimple *, vec *, vec *, int); -- cgit v1.1