diff options
Diffstat (limited to 'gcc/tree-vectorizer.h')
| -rw-r--r-- | gcc/tree-vectorizer.h | 83 |
1 files changed, 63 insertions, 20 deletions
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 01d19c7..799d5fe 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -30,6 +30,7 @@ typedef struct _slp_tree *slp_tree; #include "internal-fn.h" #include "tree-ssa-operands.h" #include "gimple-match.h" +#include "dominance.h" /* Used for naming of new temporaries. */ enum vect_var_kind { @@ -55,6 +56,14 @@ enum dr_alignment_support { dr_aligned }; +/* Define type of peeling support to indicate how peeling for alignment can help + make vectorization supported. */ +enum peeling_support { + peeling_known_supported, + peeling_maybe_supported, + peeling_unsupported +}; + /* Define type of def-use cross-iteration cycle. */ enum vect_def_type { vect_uninitialized_def = 0, @@ -265,6 +274,9 @@ struct _slp_tree { /* Whether uses of this load or feeders of this store are suitable for load/store-lanes. */ bool ldst_lanes; + /* For BB vect, flag to indicate this load node should be vectorized + as to avoid STLF fails because of related stores. */ + bool avoid_stlf_fail; int vertex; @@ -942,6 +954,13 @@ public: epilogue of loop. */ bool epil_using_partial_vectors_p; + /* True if we've decided to use peeling with versioning together, which allows + unaligned unsupported data refs to be uniformly aligned after a certain + amount of peeling (mutual alignment). Otherwise, we use versioning alone + so these data refs must be already aligned to a power-of-two boundary + without peeling. */ + bool allow_mutual_alignment; + /* The bias for len_load and len_store. For now, only 0 and -1 are supported. -1 must be used when a backend does not support len_load/len_store with a length of zero. */ @@ -966,6 +985,10 @@ public: /* Main loop IV cond. */ gcond* loop_iv_cond; + /* True if we have an unroll factor requested by the user through pragma GCC + unroll. */ + bool user_unroll; + /* True if there are no loop carried data dependencies in the loop. If loop->safelen <= 1, then this is always true, either the loop didn't have any loop carried data dependencies, or the loop is being @@ -1066,6 +1089,7 @@ public: #define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p #define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L) \ (L)->epil_using_partial_vectors_p +#define LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT(L) (L)->allow_mutual_alignment #define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias #define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor #define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor @@ -1090,6 +1114,7 @@ public: #define LOOP_VINFO_CHECK_UNEQUAL_ADDRS(L) (L)->check_unequal_addrs #define LOOP_VINFO_CHECK_NONZERO(L) (L)->check_nonzero #define LOOP_VINFO_LOWER_BOUNDS(L) (L)->lower_bounds +#define LOOP_VINFO_USER_UNROLL(L) (L)->user_unroll #define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores #define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor @@ -1689,7 +1714,7 @@ public: unsigned int outside_cost () const; unsigned int total_cost () const; unsigned int suggested_unroll_factor () const; - machine_mode suggested_epilogue_mode () const; + machine_mode suggested_epilogue_mode (int &masked) const; protected: unsigned int record_stmt_cost (stmt_vec_info, vect_cost_model_location, @@ -1713,8 +1738,13 @@ protected: unsigned int m_suggested_unroll_factor; /* The suggested mode to be used for a vectorized epilogue or VOIDmode, - determined at finish_cost. */ + determined at finish_cost. m_masked_epilogue specifies whether the + epilogue should use masked vectorization, regardless of the + --param vect-partial-vector-usage default. If -1 then the + --param setting takes precedence. If the user explicitly specified + --param vect-partial-vector-usage then that takes precedence. */ machine_mode m_suggested_epilogue_mode; + int m_masked_epilogue; /* True if finish_cost has been called. */ bool m_finished; @@ -1730,6 +1760,7 @@ vector_costs::vector_costs (vec_info *vinfo, bool costing_for_scalar) m_costs (), m_suggested_unroll_factor(1), m_suggested_epilogue_mode(VOIDmode), + m_masked_epilogue (-1), m_finished (false) { } @@ -1790,9 +1821,10 @@ vector_costs::suggested_unroll_factor () const /* Return the suggested epilogue mode. */ inline machine_mode -vector_costs::suggested_epilogue_mode () const +vector_costs::suggested_epilogue_mode (int &masked_p) const { gcc_checking_assert (m_finished); + masked_p = m_masked_epilogue; return m_suggested_epilogue_mode; } @@ -1870,11 +1902,25 @@ vect_orig_stmt (stmt_vec_info stmt_info) inline stmt_vec_info get_later_stmt (stmt_vec_info stmt1_info, stmt_vec_info stmt2_info) { - if (gimple_uid (vect_orig_stmt (stmt1_info)->stmt) - > gimple_uid (vect_orig_stmt (stmt2_info)->stmt)) + gimple *stmt1 = vect_orig_stmt (stmt1_info)->stmt; + gimple *stmt2 = vect_orig_stmt (stmt2_info)->stmt; + if (gimple_bb (stmt1) == gimple_bb (stmt2)) + { + if (gimple_uid (stmt1) > gimple_uid (stmt2)) + return stmt1_info; + else + return stmt2_info; + } + /* ??? We should be really calling this function only with stmts + in the same BB but we can recover if there's a domination + relationship between them. */ + else if (dominated_by_p (CDI_DOMINATORS, + gimple_bb (stmt1), gimple_bb (stmt2))) return stmt1_info; - else + else if (dominated_by_p (CDI_DOMINATORS, + gimple_bb (stmt2), gimple_bb (stmt1))) return stmt2_info; + gcc_unreachable (); } /* If STMT_INFO has been replaced by a pattern statement, return the @@ -2372,6 +2418,7 @@ extern tree get_mask_type_for_scalar_type (vec_info *, tree, unsigned int = 0); extern tree get_mask_type_for_scalar_type (vec_info *, tree, slp_tree); extern tree get_same_sized_vectype (tree, tree); extern bool vect_chooses_same_modes_p (vec_info *, machine_mode); +extern bool vect_chooses_same_modes_p (machine_mode, machine_mode); extern bool vect_get_loop_mask_type (loop_vec_info); extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *, stmt_vec_info * = NULL, gimple ** = NULL); @@ -2423,17 +2470,15 @@ record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count, STMT_VINFO_VECTYPE (stmt_info), misalign, where); } -/* Overload of record_stmt_cost with VECTYPE derived from STMT_INFO and - SLP node specified. */ +/* Overload of record_stmt_cost with VECTYPE derived from SLP node. */ inline unsigned record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count, - enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, - slp_tree node, + enum vect_cost_for_stmt kind, slp_tree node, int misalign, enum vect_cost_model_location where) { - return record_stmt_cost (body_cost_vec, count, kind, stmt_info, node, - STMT_VINFO_VECTYPE (stmt_info), misalign, where); + return record_stmt_cost (body_cost_vec, count, kind, node, + SLP_TREE_VECTYPE (node), misalign, where); } extern void vect_finish_replace_stmt (vec_info *, stmt_vec_info, gimple *); @@ -2461,8 +2506,7 @@ extern bool vect_transform_stmt (vec_info *, stmt_vec_info, slp_tree, slp_instance); extern void vect_remove_stores (vec_info *, stmt_vec_info); extern bool vect_nop_conversion_p (stmt_vec_info); -extern opt_result vect_analyze_stmt (vec_info *, stmt_vec_info, bool *, - slp_tree, +extern opt_result vect_analyze_stmt (vec_info *, slp_tree, slp_instance, stmt_vector_for_cost *); extern void vect_get_load_cost (vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, bool, @@ -2607,12 +2651,11 @@ extern bool vectorizable_induction (loop_vec_info, stmt_vec_info, stmt_vector_for_cost *); extern bool vect_transform_reduction (loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, - gimple **, slp_tree); + slp_tree); extern bool vect_transform_cycle_phi (loop_vec_info, stmt_vec_info, - gimple **, slp_tree, slp_instance); -extern bool vectorizable_lc_phi (loop_vec_info, stmt_vec_info, - gimple **, slp_tree); +extern bool vectorizable_lc_phi (loop_vec_info, stmt_vec_info, slp_tree); +extern bool vect_transform_lc_phi (loop_vec_info, stmt_vec_info, slp_tree); extern bool vectorizable_phi (vec_info *, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *); extern bool vectorizable_recurr (loop_vec_info, stmt_vec_info, @@ -2645,7 +2688,7 @@ extern bool vect_slp_analyze_operations (vec_info *); extern void vect_schedule_slp (vec_info *, const vec<slp_instance> &); extern opt_result vect_analyze_slp (vec_info *, unsigned, bool); extern bool vect_make_slp_decision (loop_vec_info); -extern void vect_detect_hybrid_slp (loop_vec_info); +extern bool vect_detect_hybrid_slp (loop_vec_info); extern void vect_optimize_slp (vec_info *); extern void vect_gather_slp_loads (vec_info *); extern tree vect_get_slp_scalar_def (slp_tree, unsigned); @@ -2665,7 +2708,7 @@ extern void duplicate_and_interleave (vec_info *, gimple_seq *, tree, extern int vect_get_place_in_interleaving_chain (stmt_vec_info, stmt_vec_info); extern slp_tree vect_create_new_slp_node (unsigned, tree_code); extern void vect_free_slp_tree (slp_tree); -extern bool compatible_calls_p (gcall *, gcall *); +extern bool compatible_calls_p (gcall *, gcall *, bool); extern int vect_slp_child_index_for_operand (const gimple *, int op, bool); extern tree prepare_vec_mask (loop_vec_info, tree, tree, tree, |