diff options
Diffstat (limited to 'gcc/tree-vect-slp.c')
| -rw-r--r-- | gcc/tree-vect-slp.c | 145 |
1 files changed, 67 insertions, 78 deletions
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index 7ad7c12..161655a 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -834,20 +834,21 @@ vect_build_slp_tree_1 (vec_info *vinfo, The value returned is the depth in the SLP tree where a mismatch was found. */ -static bool +static slp_tree vect_build_slp_tree (vec_info *vinfo, - slp_tree *node, unsigned int group_size, + vec<gimple *> stmts, unsigned int group_size, unsigned int *max_nunits, vec<slp_tree> *loads, bool *matches, unsigned *npermutes, unsigned *tree_size, unsigned max_tree_size) { - unsigned nops, i, this_tree_size = 0; + unsigned nops, i, this_tree_size = 0, this_max_nunits = *max_nunits; gimple *stmt; + slp_tree node; matches[0] = false; - stmt = SLP_TREE_SCALAR_STMTS (*node)[0]; + stmt = stmts[0]; if (is_gimple_call (stmt)) nops = gimple_call_num_args (stmt); else if (is_gimple_assign (stmt)) @@ -857,27 +858,28 @@ vect_build_slp_tree (vec_info *vinfo, nops++; } else - return false; + return NULL; bool two_operators = false; if (!vect_build_slp_tree_1 (vinfo, - SLP_TREE_SCALAR_STMTS (*node), group_size, nops, - max_nunits, matches, &two_operators)) - return false; - SLP_TREE_TWO_OPERATORS (*node) = two_operators; + stmts, group_size, nops, + &this_max_nunits, matches, &two_operators)) + return NULL; /* If the SLP node is a load, terminate the recursion. */ if (STMT_VINFO_GROUPED_ACCESS (vinfo_for_stmt (stmt)) && DR_IS_READ (STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt)))) { - loads->safe_push (*node); - return true; + *max_nunits = this_max_nunits; + node = vect_create_new_slp_node (stmts); + loads->safe_push (node); + return node; } /* Get at the operands, verifying they are compatible. */ vec<slp_oprnd_info> oprnds_info = vect_create_oprnd_info (nops, group_size); slp_oprnd_info oprnd_info; - FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (*node), i, stmt) + FOR_EACH_VEC_ELT (stmts, i, stmt) { switch (vect_get_and_check_slp_defs (vinfo, stmt, i, &oprnds_info)) { @@ -886,7 +888,7 @@ vect_build_slp_tree (vec_info *vinfo, case -1: matches[0] = false; vect_free_oprnd_info (oprnds_info); - return false; + return NULL; case 1: matches[i] = false; break; @@ -896,43 +898,43 @@ vect_build_slp_tree (vec_info *vinfo, if (!matches[i]) { vect_free_oprnd_info (oprnds_info); - return false; + return NULL; } - stmt = SLP_TREE_SCALAR_STMTS (*node)[0]; + auto_vec<slp_tree, 4> children; + auto_vec<slp_tree> this_loads; + + stmt = stmts[0]; /* Create SLP_TREE nodes for the definition node/s. */ FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info) { slp_tree child; - unsigned old_nloads = loads->length (); - unsigned old_max_nunits = *max_nunits; + unsigned old_nloads = this_loads.length (); + unsigned old_tree_size = this_tree_size; + unsigned int j; if (oprnd_info->first_dt != vect_internal_def) continue; if (++this_tree_size > max_tree_size) { + FOR_EACH_VEC_ELT (children, j, child) + vect_free_slp_tree (child); vect_free_oprnd_info (oprnds_info); - return false; + return NULL; } - child = vect_create_new_slp_node (oprnd_info->def_stmts); - if (!child) - { - vect_free_oprnd_info (oprnds_info); - return false; - } - - if (vect_build_slp_tree (vinfo, &child, - group_size, max_nunits, loads, matches, - npermutes, &this_tree_size, max_tree_size)) + if ((child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts, + group_size, &this_max_nunits, + &this_loads, matches, npermutes, + &this_tree_size, + max_tree_size)) != NULL) { /* If we have all children of child built up from scalars then just throw that away and build it up this node from scalars. */ if (!SLP_TREE_CHILDREN (child).is_empty ()) { - unsigned int j; slp_tree grandchild; FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild) @@ -941,8 +943,8 @@ vect_build_slp_tree (vec_info *vinfo, if (!grandchild) { /* Roll back. */ - *max_nunits = old_max_nunits; - loads->truncate (old_nloads); + this_loads.truncate (old_nloads); + this_tree_size = old_tree_size; FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild) vect_free_slp_tree (grandchild); SLP_TREE_CHILDREN (child).truncate (0); @@ -952,13 +954,13 @@ vect_build_slp_tree (vec_info *vinfo, "scalars instead\n"); oprnd_info->def_stmts = vNULL; SLP_TREE_DEF_TYPE (child) = vect_external_def; - SLP_TREE_CHILDREN (*node).quick_push (child); + children.safe_push (child); continue; } } oprnd_info->def_stmts = vNULL; - SLP_TREE_CHILDREN (*node).quick_push (child); + children.safe_push (child); continue; } @@ -977,21 +979,12 @@ vect_build_slp_tree (vec_info *vinfo, scalar version. */ && !is_pattern_stmt_p (vinfo_for_stmt (stmt))) { - unsigned int j; - slp_tree grandchild; - - /* Roll back. */ - *max_nunits = old_max_nunits; - loads->truncate (old_nloads); - FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild) - vect_free_slp_tree (grandchild); - SLP_TREE_CHILDREN (child).truncate (0); - dump_printf_loc (MSG_NOTE, vect_location, "Building vector operands from scalars\n"); - oprnd_info->def_stmts = vNULL; + child = vect_create_new_slp_node (oprnd_info->def_stmts); SLP_TREE_DEF_TYPE (child) = vect_external_def; - SLP_TREE_CHILDREN (*node).quick_push (child); + children.safe_push (child); + oprnd_info->def_stmts = vNULL; continue; } @@ -1007,23 +1000,13 @@ vect_build_slp_tree (vec_info *vinfo, && oprnds_info[1]->first_dt == vect_internal_def && is_gimple_assign (stmt) && commutative_tree_code (gimple_assign_rhs_code (stmt)) - && !SLP_TREE_TWO_OPERATORS (*node) + && ! two_operators /* Do so only if the number of not successful permutes was nor more than a cut-ff as re-trying the recursive match on possibly each level of the tree would expose exponential behavior. */ && *npermutes < 4) { - unsigned int j; - slp_tree grandchild; - - /* Roll back. */ - *max_nunits = old_max_nunits; - loads->truncate (old_nloads); - FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild) - vect_free_slp_tree (grandchild); - SLP_TREE_CHILDREN (child).truncate (0); - /* Swap mismatched definition stmts. */ dump_printf_loc (MSG_NOTE, vect_location, "Re-trying with swapped operands of stmts "); @@ -1037,10 +1020,11 @@ vect_build_slp_tree (vec_info *vinfo, dump_printf (MSG_NOTE, "\n"); /* And try again with scratch 'matches' ... */ bool *tem = XALLOCAVEC (bool, group_size); - if (vect_build_slp_tree (vinfo, &child, - group_size, max_nunits, loads, - tem, npermutes, &this_tree_size, - max_tree_size)) + if ((child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts, + group_size, &this_max_nunits, + &this_loads, tem, npermutes, + &this_tree_size, + max_tree_size)) != NULL) { /* ... so if successful we can apply the operand swapping to the GIMPLE IL. This is necessary because for example @@ -1050,12 +1034,12 @@ vect_build_slp_tree (vec_info *vinfo, we'll continue to process swapped operand two. */ for (j = 0; j < group_size; ++j) { - gimple *stmt = SLP_TREE_SCALAR_STMTS (*node)[j]; + gimple *stmt = stmts[j]; gimple_set_plf (stmt, GF_PLF_1, false); } for (j = 0; j < group_size; ++j) { - gimple *stmt = SLP_TREE_SCALAR_STMTS (*node)[j]; + gimple *stmt = stmts[j]; if (!matches[j]) { /* Avoid swapping operands twice. */ @@ -1070,7 +1054,7 @@ vect_build_slp_tree (vec_info *vinfo, if (flag_checking) for (j = 0; j < group_size; ++j) { - gimple *stmt = SLP_TREE_SCALAR_STMTS (*node)[j]; + gimple *stmt = stmts[j]; gcc_assert (gimple_plf (stmt, GF_PLF_1) == ! matches[j]); } @@ -1087,8 +1071,8 @@ vect_build_slp_tree (vec_info *vinfo, if (!grandchild) { /* Roll back. */ - *max_nunits = old_max_nunits; - loads->truncate (old_nloads); + this_loads.truncate (old_nloads); + this_tree_size = old_tree_size; FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild) vect_free_slp_tree (grandchild); SLP_TREE_CHILDREN (child).truncate (0); @@ -1098,30 +1082,37 @@ vect_build_slp_tree (vec_info *vinfo, "scalars instead\n"); oprnd_info->def_stmts = vNULL; SLP_TREE_DEF_TYPE (child) = vect_external_def; - SLP_TREE_CHILDREN (*node).quick_push (child); + children.safe_push (child); continue; } } oprnd_info->def_stmts = vNULL; - SLP_TREE_CHILDREN (*node).quick_push (child); + children.safe_push (child); continue; } ++*npermutes; } - oprnd_info->def_stmts = vNULL; - vect_free_slp_tree (child); + gcc_assert (child == NULL); + FOR_EACH_VEC_ELT (children, j, child) + vect_free_slp_tree (child); vect_free_oprnd_info (oprnds_info); - return false; + return NULL; } + vect_free_oprnd_info (oprnds_info); + if (tree_size) *tree_size += this_tree_size; + *max_nunits = this_max_nunits; + loads->safe_splice (this_loads); - vect_free_oprnd_info (oprnds_info); - return true; + node = vect_create_new_slp_node (stmts); + SLP_TREE_TWO_OPERATORS (node) = two_operators; + SLP_TREE_CHILDREN (node).splice (children); + return node; } /* Dump a slp tree NODE using flags specified in DUMP_KIND. */ @@ -1733,16 +1724,14 @@ vect_analyze_slp_instance (vec_info *vinfo, scalar_stmts.safe_push (next); } - node = vect_create_new_slp_node (scalar_stmts); - loads.create (group_size); /* Build the tree for the SLP instance. */ bool *matches = XALLOCAVEC (bool, group_size); unsigned npermutes = 0; - if (vect_build_slp_tree (vinfo, &node, group_size, - &max_nunits, &loads, - matches, &npermutes, NULL, max_tree_size)) + if ((node = vect_build_slp_tree (vinfo, scalar_stmts, group_size, + &max_nunits, &loads, matches, &npermutes, + NULL, max_tree_size)) != NULL) { /* Calculate the unrolling factor based on the smallest type. */ if (max_nunits > nunits) @@ -1864,7 +1853,7 @@ vect_analyze_slp_instance (vec_info *vinfo, /* Failed to SLP. */ /* Free the allocated memory. */ - vect_free_slp_tree (node); + scalar_stmts.release (); loads.release (); /* For basic block SLP, try to break the group up into multiples of the |