aboutsummaryrefslogtreecommitdiff
path: root/gcc/tree-vectorizer.h
diff options
context:
space:
mode:
Diffstat (limited to 'gcc/tree-vectorizer.h')
-rw-r--r--gcc/tree-vectorizer.h183
1 files changed, 87 insertions, 96 deletions
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 799d5fe..0a75ee1 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -173,6 +173,8 @@ struct vect_scalar_ops_slice_hash : typed_noop_remove<vect_scalar_ops_slice>
/* Describes how we're going to vectorize an individual load or store,
or a group of loads or stores. */
enum vect_memory_access_type {
+ VMAT_UNINITIALIZED,
+
/* An access to an invariant address. This is used only for loads. */
VMAT_INVARIANT,
@@ -184,11 +186,6 @@ enum vect_memory_access_type {
of invariants. */
VMAT_CONTIGUOUS_DOWN,
- /* A simple contiguous access in which the elements need to be permuted
- after loading or before storing. Only used for loop vectorization;
- SLP uses separate permutes. */
- VMAT_CONTIGUOUS_PERMUTE,
-
/* A simple contiguous access in which the elements need to be reversed
after loading or before storing. */
VMAT_CONTIGUOUS_REVERSE,
@@ -210,6 +207,32 @@ enum vect_memory_access_type {
VMAT_GATHER_SCATTER
};
+/*-----------------------------------------------------------------*/
+/* Info on vectorized defs. */
+/*-----------------------------------------------------------------*/
+enum stmt_vec_info_type {
+ undef_vec_info_type = 0,
+ load_vec_info_type,
+ store_vec_info_type,
+ shift_vec_info_type,
+ op_vec_info_type,
+ call_vec_info_type,
+ call_simd_clone_vec_info_type,
+ assignment_vec_info_type,
+ condition_vec_info_type,
+ comparison_vec_info_type,
+ reduc_vec_info_type,
+ induc_vec_info_type,
+ type_promotion_vec_info_type,
+ type_demotion_vec_info_type,
+ type_conversion_vec_info_type,
+ cycle_phi_info_type,
+ lc_phi_info_type,
+ phi_info_type,
+ recurr_info_type,
+ loop_exit_ctrl_vec_info_type
+};
+
/************************************************************************
SLP
************************************************************************/
@@ -218,6 +241,23 @@ typedef auto_vec<std::pair<unsigned, unsigned>, 16> auto_lane_permutation_t;
typedef vec<unsigned> load_permutation_t;
typedef auto_vec<unsigned, 16> auto_load_permutation_t;
+struct vect_data {
+ virtual ~vect_data () = default;
+};
+
+/* Analysis data from vectorizable_simd_clone_call for
+ call_simd_clone_vec_info_type. */
+struct vect_simd_clone_data : vect_data {
+ virtual ~vect_simd_clone_data () = default;
+ vect_simd_clone_data () = default;
+ vect_simd_clone_data (vect_simd_clone_data &&other) = default;
+
+ /* Selected SIMD clone's function info. First vector element
+ is SIMD clone's function decl, followed by a pair of trees (base + step)
+ for linear arguments (pair of NULLs for other arguments). */
+ auto_vec<tree> simd_clone_info;
+};
+
/* A computation tree of an SLP instance. Each node corresponds to a group of
stmts to be packed in a SIMD stmt. */
struct _slp_tree {
@@ -246,11 +286,6 @@ struct _slp_tree {
denotes the number of output lanes. */
lane_permutation_t lane_permutation;
- /* Selected SIMD clone's function info. First vector element
- is SIMD clone's function decl, followed by a pair of trees (base + step)
- for linear arguments (pair of NULLs for other arguments). */
- vec<tree> simd_clone_info;
-
tree vectype;
/* Vectorized defs. */
vec<tree> vec_defs;
@@ -284,6 +319,14 @@ struct _slp_tree {
for loop vectorization. */
vect_memory_access_type memory_access_type;
+ /* The kind of operation as determined by analysis and optional
+ kind specific data. */
+ enum stmt_vec_info_type type;
+ vect_data *data;
+
+ template <class T>
+ T& get_data (T& else_) { return data ? *static_cast <T *> (data) : else_; }
+
/* If not NULL this is a cached failed SLP discovery attempt with
the lanes that failed during SLP discovery as 'false'. This is
a copy of the matches array. */
@@ -362,13 +405,13 @@ public:
#define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size
#define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation
#define SLP_TREE_LANE_PERMUTATION(S) (S)->lane_permutation
-#define SLP_TREE_SIMD_CLONE_INFO(S) (S)->simd_clone_info
#define SLP_TREE_DEF_TYPE(S) (S)->def_type
#define SLP_TREE_VECTYPE(S) (S)->vectype
#define SLP_TREE_REPRESENTATIVE(S) (S)->representative
#define SLP_TREE_LANES(S) (S)->lanes
#define SLP_TREE_CODE(S) (S)->code
#define SLP_TREE_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type
+#define SLP_TREE_TYPE(S) (S)->type
enum vect_partial_vector_style {
vect_partial_vectors_none,
@@ -1168,6 +1211,10 @@ public:
|| LOOP_REQUIRES_VERSIONING_FOR_NITERS (L) \
|| LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L))
+#define LOOP_VINFO_USE_VERSIONING_WITHOUT_PEELING(L) \
+ ((L)->may_misalign_stmts.length () > 0 \
+ && !LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (L))
+
#define LOOP_VINFO_NITERS_KNOWN_P(L) \
(tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0)
@@ -1216,32 +1263,6 @@ public:
#define BB_VINFO_DATAREFS(B) (B)->shared->datarefs
#define BB_VINFO_DDRS(B) (B)->shared->ddrs
-/*-----------------------------------------------------------------*/
-/* Info on vectorized defs. */
-/*-----------------------------------------------------------------*/
-enum stmt_vec_info_type {
- undef_vec_info_type = 0,
- load_vec_info_type,
- store_vec_info_type,
- shift_vec_info_type,
- op_vec_info_type,
- call_vec_info_type,
- call_simd_clone_vec_info_type,
- assignment_vec_info_type,
- condition_vec_info_type,
- comparison_vec_info_type,
- reduc_vec_info_type,
- induc_vec_info_type,
- type_promotion_vec_info_type,
- type_demotion_vec_info_type,
- type_conversion_vec_info_type,
- cycle_phi_info_type,
- lc_phi_info_type,
- phi_info_type,
- recurr_info_type,
- loop_exit_ctrl_vec_info_type
-};
-
/* Indicates whether/how a variable is used in the scope of loop/basic
block. */
enum vect_relevant {
@@ -1334,8 +1355,6 @@ typedef struct data_reference *dr_p;
class _stmt_vec_info {
public:
- enum stmt_vec_info_type type;
-
/* Indicates whether this stmts is part of a computation whose result is
used outside the loop. */
bool live;
@@ -1359,9 +1378,6 @@ public:
/* The vector type to be used for the LHS of this statement. */
tree vectype;
- /* The vectorized stmts. */
- vec<gimple *> vec_stmts;
-
/* The following is relevant only for stmts that contain a non-scalar
data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
at most one such data-ref. */
@@ -1398,11 +1414,6 @@ public:
pattern statement. */
gimple_seq pattern_def_seq;
- /* Selected SIMD clone's function info. First vector element
- is SIMD clone's function decl, followed by a pair of trees (base + step)
- for linear arguments (pair of NULLs for other arguments). */
- vec<tree> simd_clone_info;
-
/* Classify the def of this stmt. */
enum vect_def_type def_type;
@@ -1441,10 +1452,6 @@ public:
/* For both loads and stores. */
unsigned simd_lane_access_p : 3;
- /* Classifies how the load or store is going to be implemented
- for loop vectorization. */
- vect_memory_access_type memory_access_type;
-
/* For INTEGER_INDUC_COND_REDUCTION, the initial value to be used. */
tree induc_cond_initial_val;
@@ -1557,6 +1564,10 @@ struct gather_scatter_info {
/* The loop-invariant base value. */
tree base;
+ /* The TBBA alias pointer the value of which determines the alignment
+ of the scalar accesses. */
+ tree alias_ptr;
+
/* The original scalar offset, which is a non-loop-invariant SSA_NAME. */
tree offset;
@@ -1564,9 +1575,6 @@ struct gather_scatter_info {
being added to the base. */
int scale;
- /* The definition type for the vectorized offset. */
- enum vect_def_type offset_dt;
-
/* The type of the vectorized offset. */
tree offset_vectype;
@@ -1578,17 +1586,14 @@ struct gather_scatter_info {
};
/* Access Functions. */
-#define STMT_VINFO_TYPE(S) (S)->type
#define STMT_VINFO_STMT(S) (S)->stmt
#define STMT_VINFO_RELEVANT(S) (S)->relevant
#define STMT_VINFO_LIVE_P(S) (S)->live
#define STMT_VINFO_VECTYPE(S) (S)->vectype
-#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts
#define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
#define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0)
#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
#define STMT_VINFO_STRIDED_P(S) (S)->strided_p
-#define STMT_VINFO_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type
#define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
#define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
#define STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT(S) (S)->reduc_epilogue_adjustment
@@ -1614,7 +1619,6 @@ struct gather_scatter_info {
#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
#define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
-#define STMT_VINFO_SIMD_CLONE_INFO(S) (S)->simd_clone_info
#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
#define STMT_VINFO_GROUPED_ACCESS(S) \
((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
@@ -1655,6 +1659,14 @@ struct gather_scatter_info {
#define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
#define STMT_SLP_TYPE(S) (S)->slp_type
+#define GATHER_SCATTER_LEGACY_P(info) ((info).decl != NULL_TREE \
+ && (info).ifn == IFN_LAST)
+#define GATHER_SCATTER_IFN_P(info) ((info).decl == NULL_TREE \
+ && (info).ifn != IFN_LAST)
+#define GATHER_SCATTER_EMULATED_P(info) ((info).decl == NULL_TREE \
+ && (info).ifn == IFN_LAST)
+
+
/* Contains the scalar or vector costs for a vec_info. */
class vector_costs
{
@@ -1715,6 +1727,7 @@ public:
unsigned int total_cost () const;
unsigned int suggested_unroll_factor () const;
machine_mode suggested_epilogue_mode (int &masked) const;
+ bool costing_for_scalar () const { return m_costing_for_scalar; }
protected:
unsigned int record_stmt_cost (stmt_vec_info, vect_cost_model_location,
@@ -1997,6 +2010,13 @@ add_stmt_cost (vector_costs *costs, int count,
tree vectype, int misalign,
enum vect_cost_model_location where)
{
+ /* Even though a vector type might be set on stmt do not pass that on when
+ costing the scalar IL. A SLP node shouldn't have been recorded. */
+ if (costs->costing_for_scalar ())
+ {
+ vectype = NULL_TREE;
+ gcc_checking_assert (node == NULL);
+ }
unsigned cost = costs->add_stmt_cost (count, kind, stmt_info, node, vectype,
misalign, where);
if (dump_file && (dump_flags & TDF_DETAILS))
@@ -2422,10 +2442,7 @@ extern bool vect_chooses_same_modes_p (machine_mode, machine_mode);
extern bool vect_get_loop_mask_type (loop_vec_info);
extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
stmt_vec_info * = NULL, gimple ** = NULL);
-extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
- tree *, stmt_vec_info * = NULL,
- gimple ** = NULL);
-extern bool vect_is_simple_use (vec_info *, stmt_vec_info, slp_tree,
+extern bool vect_is_simple_use (vec_info *, slp_tree,
unsigned, tree *, slp_tree *,
enum vect_def_type *,
tree *, stmt_vec_info * = NULL);
@@ -2486,18 +2503,11 @@ extern void vect_finish_stmt_generation (vec_info *, stmt_vec_info, gimple *,
gimple_stmt_iterator *);
extern opt_result vect_mark_stmts_to_be_vectorized (loop_vec_info, bool *);
extern tree vect_get_store_rhs (stmt_vec_info);
-void vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info, unsigned,
- tree op, vec<tree> *, tree = NULL);
-void vect_get_vec_defs (vec_info *, stmt_vec_info, slp_tree, unsigned,
+void vect_get_vec_defs (vec_info *, slp_tree,
tree, vec<tree> *,
tree = NULL, vec<tree> * = NULL,
tree = NULL, vec<tree> * = NULL,
tree = NULL, vec<tree> * = NULL);
-void vect_get_vec_defs (vec_info *, stmt_vec_info, slp_tree, unsigned,
- tree, tree, vec<tree> *,
- tree = NULL, tree = NULL, vec<tree> * = NULL,
- tree = NULL, tree = NULL, vec<tree> * = NULL,
- tree = NULL, tree = NULL, vec<tree> * = NULL);
extern tree vect_init_vector (vec_info *, stmt_vec_info, tree, tree,
gimple_stmt_iterator *);
extern tree vect_get_slp_vect_def (slp_tree, unsigned);
@@ -2534,7 +2544,8 @@ extern bool ref_within_array_bound (gimple *, tree);
/* In tree-vect-data-refs.cc. */
extern bool vect_can_force_dr_alignment_p (const_tree, poly_uint64);
extern enum dr_alignment_support vect_supportable_dr_alignment
- (vec_info *, dr_vec_info *, tree, int);
+ (vec_info *, dr_vec_info *, tree, int,
+ gather_scatter_info * = nullptr);
extern tree vect_get_smallest_scalar_type (stmt_vec_info, tree);
extern opt_result vect_analyze_data_ref_dependences (loop_vec_info, unsigned int *);
extern bool vect_slp_analyze_instance_dependence (vec_info *, slp_instance);
@@ -2552,7 +2563,7 @@ extern bool vect_check_gather_scatter (stmt_vec_info, loop_vec_info,
extern opt_result vect_find_stmt_data_reference (loop_p, gimple *,
vec<data_reference_p> *,
vec<int> *, int);
-extern opt_result vect_analyze_data_refs (vec_info *, poly_uint64 *, bool *);
+extern opt_result vect_analyze_data_refs (vec_info *, bool *);
extern void vect_record_base_alignments (vec_info *);
extern tree vect_create_data_ref_ptr (vec_info *,
stmt_vec_info, tree, class loop *, tree,
@@ -2568,17 +2579,10 @@ extern internal_fn vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT, boo
extern bool vect_grouped_load_supported (tree, bool, unsigned HOST_WIDE_INT);
extern internal_fn vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT,
bool, vec<int> * = nullptr);
-extern void vect_permute_store_chain (vec_info *, vec<tree> &,
- unsigned int, stmt_vec_info,
- gimple_stmt_iterator *, vec<tree> *);
extern tree vect_setup_realignment (vec_info *,
stmt_vec_info, gimple_stmt_iterator *,
tree *, enum dr_alignment_support, tree,
class loop **);
-extern void vect_transform_grouped_load (vec_info *, stmt_vec_info, vec<tree>,
- int, gimple_stmt_iterator *);
-extern void vect_record_grouped_load_vectors (vec_info *,
- stmt_vec_info, vec<tree>);
extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
extern tree vect_get_new_ssa_name (tree, enum vect_var_kind,
const char * = NULL);
@@ -2647,8 +2651,7 @@ extern bool vectorizable_reduction (loop_vec_info, stmt_vec_info,
slp_tree, slp_instance,
stmt_vector_for_cost *);
extern bool vectorizable_induction (loop_vec_info, stmt_vec_info,
- gimple **, slp_tree,
- stmt_vector_for_cost *);
+ slp_tree, stmt_vector_for_cost *);
extern bool vect_transform_reduction (loop_vec_info, stmt_vec_info,
gimple_stmt_iterator *,
slp_tree);
@@ -2656,12 +2659,12 @@ extern bool vect_transform_cycle_phi (loop_vec_info, stmt_vec_info,
slp_tree, slp_instance);
extern bool vectorizable_lc_phi (loop_vec_info, stmt_vec_info, slp_tree);
extern bool vect_transform_lc_phi (loop_vec_info, stmt_vec_info, slp_tree);
-extern bool vectorizable_phi (vec_info *, stmt_vec_info, gimple **, slp_tree,
+extern bool vectorizable_phi (vec_info *, stmt_vec_info, slp_tree,
stmt_vector_for_cost *);
extern bool vectorizable_recurr (loop_vec_info, stmt_vec_info,
- gimple **, slp_tree, stmt_vector_for_cost *);
+ slp_tree, stmt_vector_for_cost *);
extern bool vectorizable_early_exit (vec_info *, stmt_vec_info,
- gimple_stmt_iterator *, gimple **,
+ gimple_stmt_iterator *,
slp_tree, stmt_vector_for_cost *);
extern bool vect_emulated_vector_p (tree);
extern bool vect_can_vectorize_without_simd_p (tree_code);
@@ -2839,18 +2842,6 @@ vect_is_reduction (stmt_vec_info stmt_info)
return STMT_VINFO_REDUC_IDX (stmt_info) >= 0;
}
-/* Returns the memory acccess type being used to vectorize the statement. If
- SLP this is read from NODE, otherwise it's read from the STMT_VINFO. */
-
-inline vect_memory_access_type
-vect_mem_access_type (stmt_vec_info stmt_info, slp_tree node)
-{
- if (node)
- return SLP_TREE_MEMORY_ACCESS_TYPE (node);
- else
- return STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info);
-}
-
/* If STMT_INFO describes a reduction, return the vect_reduction_type
of the reduction it describes, otherwise return -1. */
inline int
generated by cgit v1.1 at 2025-08-03 14:05:02 +0000