diff options
Diffstat (limited to 'gcc/tree-vect-slp.c')
-rw-r--r-- | gcc/tree-vect-slp.c | 100 |
1 files changed, 68 insertions, 32 deletions
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index 5d11552..478343e 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -94,6 +94,7 @@ vect_free_slp_instance (slp_instance instance) vect_free_slp_tree (SLP_INSTANCE_TREE (instance)); VEC_free (int, heap, SLP_INSTANCE_LOAD_PERMUTATION (instance)); VEC_free (slp_tree, heap, SLP_INSTANCE_LOADS (instance)); + VEC_free (stmt_info_for_cost, heap, SLP_INSTANCE_STMT_COST_VEC (instance)); } @@ -122,7 +123,6 @@ vect_create_new_slp_node (VEC (gimple, heap) *scalar_stmts) SLP_TREE_VEC_STMTS (node) = NULL; SLP_TREE_CHILDREN (node) = VEC_alloc (slp_void_p, heap, nops); SLP_TREE_OUTSIDE_OF_LOOP_COST (node) = 0; - SLP_TREE_INSIDE_OF_LOOP_COST (node) = 0; return node; } @@ -179,7 +179,8 @@ static bool vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, slp_tree slp_node, gimple stmt, int ncopies_for_cost, bool first, - VEC (slp_oprnd_info, heap) **oprnds_info) + VEC (slp_oprnd_info, heap) **oprnds_info, + stmt_vector_for_cost *stmt_cost_vec) { tree oprnd; unsigned int i, number_of_oprnds; @@ -320,7 +321,7 @@ vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, if (REFERENCE_CLASS_P (lhs)) /* Store. */ vect_model_store_cost (stmt_info, ncopies_for_cost, false, - dt, slp_node); + dt, slp_node, stmt_cost_vec); else { enum vect_def_type dts[2]; @@ -329,7 +330,7 @@ vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, /* Not memory operation (we don't call this function for loads). */ vect_model_simple_cost (stmt_info, ncopies_for_cost, dts, - slp_node); + slp_node, stmt_cost_vec); } } } @@ -446,12 +447,12 @@ vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, static bool vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, - slp_tree *node, unsigned int group_size, - int *inside_cost, int *outside_cost, + slp_tree *node, unsigned int group_size, int *outside_cost, int ncopies_for_cost, unsigned int *max_nunits, VEC (int, heap) **load_permutation, VEC (slp_tree, heap) **loads, - unsigned int vectorization_factor, bool *loads_permuted) + unsigned int vectorization_factor, bool *loads_permuted, + stmt_vector_for_cost *stmt_cost_vec) { unsigned int i; VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (*node); @@ -470,7 +471,7 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, HOST_WIDE_INT dummy; bool permutation = false; unsigned int load_place; - gimple first_load, prev_first_load = NULL; + gimple first_load = NULL, prev_first_load = NULL, old_first_load = NULL; VEC (slp_oprnd_info, heap) *oprnds_info; unsigned int nops; slp_oprnd_info oprnd_info; @@ -711,7 +712,8 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, /* Store. */ if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node, stmt, ncopies_for_cost, - (i == 0), &oprnds_info)) + (i == 0), &oprnds_info, + stmt_cost_vec)) { vect_free_oprnd_info (&oprnds_info); return false; @@ -754,6 +756,7 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, return false; } + old_first_load = first_load; first_load = GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)); if (prev_first_load) { @@ -778,7 +781,9 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, else prev_first_load = first_load; - if (first_load == stmt) + /* In some cases a group of loads is just the same load + repeated N times. Only analyze its cost once. */ + if (first_load == stmt && old_first_load != first_load) { first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt)); if (vect_supportable_dr_alignment (first_dr, false) @@ -797,7 +802,8 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, /* Analyze costs (for the first stmt in the group). */ vect_model_load_cost (vinfo_for_stmt (stmt), - ncopies_for_cost, false, *node); + ncopies_for_cost, false, *node, + stmt_cost_vec); } /* Store the place of this load in the interleaving chain. In @@ -871,7 +877,7 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, /* Find the def-stmts. */ if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node, stmt, ncopies_for_cost, (i == 0), - &oprnds_info)) + &oprnds_info, stmt_cost_vec)) { vect_free_oprnd_info (&oprnds_info); return false; @@ -880,7 +886,6 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, } /* Add the costs of the node to the overall instance costs. */ - *inside_cost += SLP_TREE_INSIDE_OF_LOOP_COST (*node); *outside_cost += SLP_TREE_OUTSIDE_OF_LOOP_COST (*node); /* Grouped loads were reached - stop the recursion. */ @@ -889,11 +894,10 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, VEC_safe_push (slp_tree, heap, *loads, *node); if (permutation) { - + gimple first_stmt = VEC_index (gimple, stmts, 0); *loads_permuted = true; - *inside_cost - += targetm.vectorize.builtin_vectorization_cost (vec_perm, NULL, 0) - * group_size; + (void) record_stmt_cost (stmt_cost_vec, group_size, vec_perm, + vinfo_for_stmt (first_stmt), 0); } else { @@ -919,9 +923,10 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, child = vect_create_new_slp_node (oprnd_info->def_stmts); if (!child || !vect_build_slp_tree (loop_vinfo, bb_vinfo, &child, group_size, - inside_cost, outside_cost, ncopies_for_cost, - max_nunits, load_permutation, loads, - vectorization_factor, loads_permuted)) + outside_cost, ncopies_for_cost, + max_nunits, load_permutation, loads, + vectorization_factor, loads_permuted, + stmt_cost_vec)) { if (child) oprnd_info->def_stmts = NULL; @@ -1459,13 +1464,14 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, tree vectype, scalar_type = NULL_TREE; gimple next; unsigned int vectorization_factor = 0; - int inside_cost = 0, outside_cost = 0, ncopies_for_cost, i; + int outside_cost = 0, ncopies_for_cost, i; unsigned int max_nunits = 0; VEC (int, heap) *load_permutation; VEC (slp_tree, heap) *loads; struct data_reference *dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt)); bool loads_permuted = false; VEC (gimple, heap) *scalar_stmts; + stmt_vector_for_cost stmt_cost_vec; if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt))) { @@ -1551,12 +1557,14 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, load_permutation = VEC_alloc (int, heap, group_size * group_size); loads = VEC_alloc (slp_tree, heap, group_size); + stmt_cost_vec = VEC_alloc (stmt_info_for_cost, heap, 10); /* Build the tree for the SLP instance. */ if (vect_build_slp_tree (loop_vinfo, bb_vinfo, &node, group_size, - &inside_cost, &outside_cost, ncopies_for_cost, + &outside_cost, ncopies_for_cost, &max_nunits, &load_permutation, &loads, - vectorization_factor, &loads_permuted)) + vectorization_factor, &loads_permuted, + &stmt_cost_vec)) { /* Calculate the unrolling factor based on the smallest type. */ if (max_nunits > nunits) @@ -1568,6 +1576,7 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, if (vect_print_dump_info (REPORT_SLP)) fprintf (vect_dump, "Build SLP failed: unrolling required in basic" " block SLP"); + VEC_free (stmt_info_for_cost, heap, stmt_cost_vec); return false; } @@ -1577,7 +1586,7 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, SLP_INSTANCE_GROUP_SIZE (new_instance) = group_size; SLP_INSTANCE_UNROLLING_FACTOR (new_instance) = unrolling_factor; SLP_INSTANCE_OUTSIDE_OF_LOOP_COST (new_instance) = outside_cost; - SLP_INSTANCE_INSIDE_OF_LOOP_COST (new_instance) = inside_cost; + SLP_INSTANCE_STMT_COST_VEC (new_instance) = stmt_cost_vec; SLP_INSTANCE_LOADS (new_instance) = loads; SLP_INSTANCE_FIRST_LOAD_STMT (new_instance) = NULL; SLP_INSTANCE_LOAD_PERMUTATION (new_instance) = load_permutation; @@ -1617,6 +1626,8 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, return true; } + else + VEC_free (stmt_info_for_cost, heap, stmt_cost_vec); /* Failed to SLP. */ /* Free the allocated memory. */ @@ -1812,6 +1823,7 @@ new_bb_vec_info (basic_block bb) BB_VINFO_GROUPED_STORES (res) = VEC_alloc (gimple, heap, 10); BB_VINFO_SLP_INSTANCES (res) = VEC_alloc (slp_instance, heap, 2); + BB_VINFO_TARGET_COST_DATA (res) = init_cost (NULL); bb->aux = res; return res; @@ -1846,6 +1858,7 @@ destroy_bb_vec_info (bb_vec_info bb_vinfo) free_dependence_relations (BB_VINFO_DDRS (bb_vinfo)); VEC_free (gimple, heap, BB_VINFO_GROUPED_STORES (bb_vinfo)); VEC_free (slp_instance, heap, BB_VINFO_SLP_INSTANCES (bb_vinfo)); + destroy_cost_data (BB_VINFO_TARGET_COST_DATA (bb_vinfo)); free (bb_vinfo); bb->aux = NULL; } @@ -1918,8 +1931,8 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo) { VEC (slp_instance, heap) *slp_instances = BB_VINFO_SLP_INSTANCES (bb_vinfo); slp_instance instance; - int i; - unsigned int vec_outside_cost = 0, vec_inside_cost = 0, scalar_cost = 0; + int i, j; + unsigned int vec_inside_cost = 0, vec_outside_cost = 0, scalar_cost = 0; unsigned int stmt_cost; gimple stmt; gimple_stmt_iterator si; @@ -1927,12 +1940,19 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo) stmt_vec_info stmt_info = NULL; tree dummy_type = NULL; int dummy = 0; + stmt_vector_for_cost stmt_cost_vec; + stmt_info_for_cost *ci; /* Calculate vector costs. */ FOR_EACH_VEC_ELT (slp_instance, slp_instances, i, instance) { vec_outside_cost += SLP_INSTANCE_OUTSIDE_OF_LOOP_COST (instance); - vec_inside_cost += SLP_INSTANCE_INSIDE_OF_LOOP_COST (instance); + stmt_cost_vec = SLP_INSTANCE_STMT_COST_VEC (instance); + + FOR_EACH_VEC_ELT (stmt_info_for_cost, stmt_cost_vec, j, ci) + (void) add_stmt_cost (BB_VINFO_TARGET_COST_DATA (bb_vinfo), + ci->count, ci->kind, + vinfo_for_stmt (ci->stmt), ci->misalign); } /* Calculate scalar cost. */ @@ -1961,6 +1981,9 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo) scalar_cost += stmt_cost; } + /* Complete the target-specific cost calculation. */ + vec_inside_cost = finish_cost (BB_VINFO_TARGET_COST_DATA (bb_vinfo)); + if (vect_print_dump_info (REPORT_COST)) { fprintf (vect_dump, "Cost model analysis: \n"); @@ -2072,7 +2095,7 @@ vect_slp_analyze_bb_1 (basic_block bb) vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance)); } - if (!vect_verify_datarefs_alignment (NULL, bb_vinfo)) + if (!vect_verify_datarefs_alignment (NULL, bb_vinfo)) { if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) fprintf (vect_dump, "not vectorized: unsupported alignment in basic " @@ -2175,17 +2198,30 @@ vect_slp_analyze_bb (basic_block bb) void vect_update_slp_costs_according_to_vf (loop_vec_info loop_vinfo) { - unsigned int i, vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); + unsigned int i, j, vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); VEC (slp_instance, heap) *slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo); slp_instance instance; + stmt_vector_for_cost stmt_cost_vec; + stmt_info_for_cost *si; if (vect_print_dump_info (REPORT_SLP)) fprintf (vect_dump, "=== vect_update_slp_costs_according_to_vf ==="); FOR_EACH_VEC_ELT (slp_instance, slp_instances, i, instance) - /* We assume that costs are linear in ncopies. */ - SLP_INSTANCE_INSIDE_OF_LOOP_COST (instance) *= vf - / SLP_INSTANCE_UNROLLING_FACTOR (instance); + { + /* We assume that costs are linear in ncopies. */ + int ncopies = vf / SLP_INSTANCE_UNROLLING_FACTOR (instance); + + /* Record the instance's instructions in the target cost model. + This was delayed until here because the count of instructions + isn't known beforehand. */ + stmt_cost_vec = SLP_INSTANCE_STMT_COST_VEC (instance); + + FOR_EACH_VEC_ELT (stmt_info_for_cost, stmt_cost_vec, j, si) + (void) add_stmt_cost (LOOP_VINFO_TARGET_COST_DATA (loop_vinfo), + si->count * ncopies, si->kind, + vinfo_for_stmt (si->stmt), si->misalign); + } } |