diff options
Diffstat (limited to 'gcc')
| -rw-r--r-- | gcc/ChangeLog | 18 | ||||
| -rw-r--r-- | gcc/optabs-query.h | 9 | ||||
| -rw-r--r-- | gcc/optabs-tree.c | 3 | ||||
| -rw-r--r-- | gcc/optabs.c | 49 | ||||
| -rw-r--r-- | gcc/optabs.def | 1 | ||||
| -rw-r--r-- | gcc/tree-vect-patterns.c | 30 | ||||
| -rw-r--r-- | gcc/tree-vect-slp.c | 22 | ||||
| -rw-r--r-- | gcc/tree-vect-stmts.c | 93 |
8 files changed, 159 insertions, 66 deletions
diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 2360be2..7b740a7 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,5 +1,23 @@ 2015-11-10 Ilya Enkovich <enkovich.gnu@gmail.com> + * optabs-query.h (get_vcond_mask_icode): New. + * optabs-tree.c (expand_vec_cond_expr_p): Use + get_vcond_mask_icode for VEC_COND_EXPR with mask. + * optabs.c (expand_vec_cond_mask_expr): New. + (expand_vec_cond_expr): Use get_vcond_mask_icode + when possible. + * optabs.def (vcond_mask_optab): New. + * tree-vect-patterns.c (vect_recog_bool_pattern): Don't + generate redundant comparison for COND_EXPR. + * tree-vect-stmts.c (vect_is_simple_cond): Allow SSA_NAME + as a condition. + (vectorizable_condition): Likewise. + * tree-vect-slp.c (vect_get_and_check_slp_defs): Allow + cond_exp with no embedded comparison. + (vect_build_slp_tree_1): Likewise. + +2015-11-10 Ilya Enkovich <enkovich.gnu@gmail.com> + * config/i386/sse.md (maskload<mode>): Rename to ... (maskload<mode><sseintvecmodelower>): ... this. (maskstore<mode>): Rename to ... diff --git a/gcc/optabs-query.h b/gcc/optabs-query.h index 162d2e9..48bcf7c 100644 --- a/gcc/optabs-query.h +++ b/gcc/optabs-query.h @@ -98,6 +98,15 @@ get_vcond_icode (machine_mode vmode, machine_mode cmode, bool uns) return icode; } +/* Return insn code for a conditional operator with a mask mode + MMODE resulting in a value of mode VMODE. */ + +static inline enum insn_code +get_vcond_mask_icode (machine_mode vmode, machine_mode mmode) +{ + return convert_optab_handler (vcond_mask_optab, vmode, mmode); +} + /* Enumerates the possible extraction_insn operations. */ enum extraction_pattern { EP_insv, EP_extv, EP_extzv }; diff --git a/gcc/optabs-tree.c b/gcc/optabs-tree.c index aa863cf..d887619 100644 --- a/gcc/optabs-tree.c +++ b/gcc/optabs-tree.c @@ -342,6 +342,9 @@ expand_vec_cond_expr_p (tree value_type, tree cmp_op_type) { machine_mode value_mode = TYPE_MODE (value_type); machine_mode cmp_op_mode = TYPE_MODE (cmp_op_type); + if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type)) + return get_vcond_mask_icode (TYPE_MODE (value_type), + TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing; if (GET_MODE_SIZE (value_mode) != GET_MODE_SIZE (cmp_op_mode) || GET_MODE_NUNITS (value_mode) != GET_MODE_NUNITS (cmp_op_mode) || get_vcond_icode (TYPE_MODE (value_type), TYPE_MODE (cmp_op_type), diff --git a/gcc/optabs.c b/gcc/optabs.c index f9fbfde..9b8e958 100644 --- a/gcc/optabs.c +++ b/gcc/optabs.c @@ -5426,6 +5426,38 @@ expand_vec_perm (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target) return tmp; } +/* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its + three operands. */ + +rtx +expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2, + rtx target) +{ + struct expand_operand ops[4]; + machine_mode mode = TYPE_MODE (vec_cond_type); + machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0)); + enum insn_code icode = get_vcond_mask_icode (mode, mask_mode); + rtx mask, rtx_op1, rtx_op2; + + if (icode == CODE_FOR_nothing) + return 0; + + mask = expand_normal (op0); + rtx_op1 = expand_normal (op1); + rtx_op2 = expand_normal (op2); + + mask = force_reg (GET_MODE (mask), mask); + rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1); + + create_output_operand (&ops[0], target, mode); + create_input_operand (&ops[1], rtx_op1, mode); + create_input_operand (&ops[2], rtx_op2, mode); + create_input_operand (&ops[3], mask, mask_mode); + expand_insn (icode, 4, ops); + + return ops[0].value; +} + /* Generate insns for a VEC_COND_EXPR, given its TYPE and its three operands. */ @@ -5450,11 +5482,20 @@ expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2, } else { - /* Fake op0 < 0. */ gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))); - op0a = op0; - op0b = build_zero_cst (TREE_TYPE (op0)); - tcode = LT_EXPR; + if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0))) + != CODE_FOR_nothing) + return expand_vec_cond_mask_expr (vec_cond_type, op0, op1, + op2, target); + /* Fake op0 < 0. */ + else + { + gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0))) + == MODE_VECTOR_INT); + op0a = op0; + op0b = build_zero_cst (TREE_TYPE (op0)); + tcode = LT_EXPR; + } } cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a)); unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a)); diff --git a/gcc/optabs.def b/gcc/optabs.def index c9e52a9..0ca2333 100644 --- a/gcc/optabs.def +++ b/gcc/optabs.def @@ -61,6 +61,7 @@ OPTAB_CD(vec_load_lanes_optab, "vec_load_lanes$a$b") OPTAB_CD(vec_store_lanes_optab, "vec_store_lanes$a$b") OPTAB_CD(vcond_optab, "vcond$a$b") OPTAB_CD(vcondu_optab, "vcondu$a$b") +OPTAB_CD(vcond_mask_optab, "vcond_mask_$a$b") OPTAB_CD(vec_cmp_optab, "vec_cmp$a$b") OPTAB_CD(vec_cmpu_optab, "vec_cmpu$a$b") OPTAB_CD(maskload_optab, "maskload$a$b") diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index e91c6e0..917eeb7 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -3302,7 +3302,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, else { tree type = search_type_for_mask (var, vinfo); - tree cst0, cst1, cmp, tmp; + tree cst0, cst1, tmp; if (!type) return NULL; @@ -3318,9 +3318,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, cst0 = build_int_cst (type, 0); cst1 = build_int_cst (type, 1); tmp = vect_recog_temp_ssa_var (type, NULL); - cmp = build2 (NE_EXPR, boolean_type_node, - var, build_int_cst (TREE_TYPE (var), 0)); - pattern_stmt = gimple_build_assign (tmp, COND_EXPR, cmp, cst1, cst0); + pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0); if (!useless_type_conversion_p (type, TREE_TYPE (lhs))) { @@ -3363,19 +3361,16 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, if (get_vectype_for_scalar_type (type) == NULL_TREE) return NULL; - if (check_bool_pattern (var, vinfo)) - { - rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts); - rhs = build2 (NE_EXPR, boolean_type_node, - rhs, build_int_cst (type, 0)); - } - else - rhs = build2 (NE_EXPR, boolean_type_node, - var, build_int_cst (TREE_TYPE (var), 0)), + if (!check_bool_pattern (var, vinfo)) + return NULL; + + rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts); lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL); pattern_stmt - = gimple_build_assign (lhs, COND_EXPR, rhs, + = gimple_build_assign (lhs, COND_EXPR, + build2 (NE_EXPR, boolean_type_node, + rhs, build_int_cst (type, 0)), gimple_assign_rhs2 (last_stmt), gimple_assign_rhs3 (last_stmt)); *type_out = vectype; @@ -3402,7 +3397,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, else { tree type = search_type_for_mask (var, vinfo); - tree cst0, cst1, cmp, new_vectype; + tree cst0, cst1, new_vectype; if (!type) return NULL; @@ -3415,10 +3410,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, new_vectype = get_vectype_for_scalar_type (type); rhs = vect_recog_temp_ssa_var (type, NULL); - cmp = build2 (NE_EXPR, boolean_type_node, - var, build_int_cst (TREE_TYPE (var), 0)); - pattern_stmt = gimple_build_assign (rhs, COND_EXPR, - cmp, cst1, cst0); + pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0); pattern_stmt_info = new_stmt_vec_info (pattern_stmt, vinfo); set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info); diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index fe83828..9d97140 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -226,7 +226,8 @@ vect_get_and_check_slp_defs (vec_info *vinfo, { enum tree_code code = gimple_assign_rhs_code (stmt); number_of_oprnds = gimple_num_ops (stmt) - 1; - if (gimple_assign_rhs_code (stmt) == COND_EXPR) + if (gimple_assign_rhs_code (stmt) == COND_EXPR + && COMPARISON_CLASS_P (gimple_assign_rhs1 (stmt))) { first_op_cond = true; commutative = true; @@ -447,7 +448,6 @@ vect_build_slp_tree_1 (vec_info *vinfo, machine_mode vec_mode; HOST_WIDE_INT dummy; gimple *first_load = NULL, *prev_first_load = NULL; - tree cond; /* For every stmt in NODE find its def stmt/s. */ FOR_EACH_VEC_ELT (stmts, i, stmt) @@ -492,24 +492,6 @@ vect_build_slp_tree_1 (vec_info *vinfo, return false; } - if (is_gimple_assign (stmt) - && gimple_assign_rhs_code (stmt) == COND_EXPR - && (cond = gimple_assign_rhs1 (stmt)) - && !COMPARISON_CLASS_P (cond)) - { - if (dump_enabled_p ()) - { - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "Build SLP failed: condition is not " - "comparison "); - dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0); - dump_printf (MSG_MISSED_OPTIMIZATION, "\n"); - } - /* Fatal mismatch. */ - matches[0] = false; - return false; - } - scalar_type = vect_get_smallest_scalar_type (stmt, &dummy, &dummy); vectype = get_vectype_for_scalar_type (scalar_type); if (!vectype) diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index bdf16fa..c024348 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -7168,6 +7168,19 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, tree *comp_vectype) enum vect_def_type dt; tree vectype1 = NULL_TREE, vectype2 = NULL_TREE; + /* Mask case. */ + if (TREE_CODE (cond) == SSA_NAME + && TREE_CODE (TREE_TYPE (cond)) == BOOLEAN_TYPE) + { + gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (cond); + if (!vect_is_simple_use (cond, vinfo, &lhs_def_stmt, + &dt, comp_vectype) + || !*comp_vectype + || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype)) + return false; + return true; + } + if (!COMPARISON_CLASS_P (cond)) return false; @@ -7237,6 +7250,7 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, vec<tree> vec_oprnds2 = vNULL; vec<tree> vec_oprnds3 = vNULL; tree vec_cmp_type; + bool masked = false; if (reduc_index && STMT_SLP_TYPE (stmt_info)) return false; @@ -7296,7 +7310,13 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, if (!vect_is_simple_use (else_clause, stmt_info->vinfo, &def_stmt, &dt)) return false; - vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype); + if (VECTOR_BOOLEAN_TYPE_P (comp_vectype)) + { + vec_cmp_type = comp_vectype; + masked = true; + } + else + vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype); if (vec_cmp_type == NULL_TREE) return false; @@ -7331,14 +7351,20 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, auto_vec<tree, 4> ops; auto_vec<vec<tree>, 4> vec_defs; - ops.safe_push (TREE_OPERAND (cond_expr, 0)); - ops.safe_push (TREE_OPERAND (cond_expr, 1)); + if (masked) + ops.safe_push (cond_expr); + else + { + ops.safe_push (TREE_OPERAND (cond_expr, 0)); + ops.safe_push (TREE_OPERAND (cond_expr, 1)); + } ops.safe_push (then_clause); ops.safe_push (else_clause); vect_get_slp_defs (ops, slp_node, &vec_defs, -1); vec_oprnds3 = vec_defs.pop (); vec_oprnds2 = vec_defs.pop (); - vec_oprnds1 = vec_defs.pop (); + if (!masked) + vec_oprnds1 = vec_defs.pop (); vec_oprnds0 = vec_defs.pop (); ops.release (); @@ -7347,17 +7373,28 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, else { gimple *gtemp; - vec_cond_lhs = - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), - stmt, comp_vectype); - vect_is_simple_use (TREE_OPERAND (cond_expr, 0), - loop_vinfo, >emp, &dts[0]); - - vec_cond_rhs = - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), - stmt, comp_vectype); - vect_is_simple_use (TREE_OPERAND (cond_expr, 1), - loop_vinfo, >emp, &dts[1]); + if (masked) + { + vec_cond_lhs + = vect_get_vec_def_for_operand (cond_expr, stmt, + comp_vectype); + vect_is_simple_use (cond_expr, stmt_info->vinfo, + >emp, &dts[0]); + } + else + { + vec_cond_lhs = + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), + stmt, comp_vectype); + vect_is_simple_use (TREE_OPERAND (cond_expr, 0), + loop_vinfo, >emp, &dts[0]); + + vec_cond_rhs = + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), + stmt, comp_vectype); + vect_is_simple_use (TREE_OPERAND (cond_expr, 1), + loop_vinfo, >emp, &dts[1]); + } if (reduc_index == 1) vec_then_clause = reduc_def; else @@ -7379,10 +7416,14 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, } else { - vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0], - vec_oprnds0.pop ()); - vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1], - vec_oprnds1.pop ()); + vec_cond_lhs + = vect_get_vec_def_for_stmt_copy (dts[0], + vec_oprnds0.pop ()); + if (!masked) + vec_cond_rhs + = vect_get_vec_def_for_stmt_copy (dts[1], + vec_oprnds1.pop ()); + vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2], vec_oprnds2.pop ()); vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3], @@ -7392,7 +7433,8 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, if (!slp_node) { vec_oprnds0.quick_push (vec_cond_lhs); - vec_oprnds1.quick_push (vec_cond_rhs); + if (!masked) + vec_oprnds1.quick_push (vec_cond_rhs); vec_oprnds2.quick_push (vec_then_clause); vec_oprnds3.quick_push (vec_else_clause); } @@ -7400,12 +7442,17 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, /* Arguments are ready. Create the new vector stmt. */ FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs) { - vec_cond_rhs = vec_oprnds1[i]; vec_then_clause = vec_oprnds2[i]; vec_else_clause = vec_oprnds3[i]; - vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type, - vec_cond_lhs, vec_cond_rhs); + if (masked) + vec_compare = vec_cond_lhs; + else + { + vec_cond_rhs = vec_oprnds1[i]; + vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type, + vec_cond_lhs, vec_cond_rhs); + } vec_cond_expr = build3 (VEC_COND_EXPR, vectype, vec_compare, vec_then_clause, vec_else_clause); |