From 42fd8198b4adabe3aa74e9bb98518a1d137c3aff Mon Sep 17 00:00:00 2001 From: Ilya Enkovich Date: Tue, 10 Nov 2015 11:57:34 +0000 Subject: expr.c (do_store_flag): Use expand_vec_cmp_expr for mask results. gcc/ * expr.c (do_store_flag): Use expand_vec_cmp_expr for mask results. (const_vector_mask_from_tree): New. (const_vector_from_tree): Use const_vector_mask_from_tree for boolean vectors. * optabs-query.h (get_vec_cmp_icode): New. * optabs-tree.c (expand_vec_cmp_expr_p): New. * optabs-tree.h (expand_vec_cmp_expr_p): New. * optabs.c (vector_compare_rtx): Add OPNO arg. (expand_vec_cond_expr): Adjust to vector_compare_rtx change. (expand_vec_cmp_expr): New. * optabs.def (vec_cmp_optab): New. (vec_cmpu_optab): New. * optabs.h (expand_vec_cmp_expr): New. * tree-vect-generic.c (expand_vector_comparison): Add vector comparison optabs check. * tree-vect-loop.c (vect_determine_vectorization_factor): Ignore mask operations for VF. Add mask type computation. * tree-vect-stmts.c (get_mask_type_for_scalar_type): New. (vectorizable_comparison): New. (vect_analyze_stmt): Add vectorizable_comparison. (vect_transform_stmt): Likewise. (vect_init_vector): Support boolean vector invariants. (vect_get_vec_def_for_operand): Add VECTYPE arg. (vectorizable_condition): Directly provide vectype for invariants used in comparison. * tree-vectorizer.h (get_mask_type_for_scalar_type): New. (enum vect_var_kind): Add vect_mask_var. (enum stmt_vec_info_type): Add comparison_vec_info_type. (vectorizable_comparison): New. (vect_get_vec_def_for_operand): Add VECTYPE arg. * tree-vect-data-refs.c (vect_get_new_vect_var): Support vect_mask_var. (vect_create_destination_var): Likewise. * tree-vect-patterns.c (check_bool_pattern): Check fails if we can vectorize comparison directly. (search_type_for_mask): New. (vect_recog_bool_pattern): Support cases when bool pattern check fails. * tree-vect-slp.c (vect_build_slp_tree_1): Allow comparison statements. (vect_get_constant_vectors): Support boolean vector constants. * config/i386/i386-protos.h (ix86_expand_mask_vec_cmp): New. (ix86_expand_int_vec_cmp): New. (ix86_expand_fp_vec_cmp): New. * config/i386/i386.c (ix86_expand_sse_cmp): Allow NULL for op_true and op_false. (ix86_int_cmp_code_to_pcmp_immediate): New. (ix86_fp_cmp_code_to_pcmp_immediate): New. (ix86_cmp_code_to_pcmp_immediate): New. (ix86_expand_mask_vec_cmp): New. (ix86_expand_fp_vec_cmp): New. (ix86_expand_int_sse_cmp): New. (ix86_expand_int_vcond): Use ix86_expand_int_sse_cmp. (ix86_expand_int_vec_cmp): New. (ix86_get_mask_mode): New. (TARGET_VECTORIZE_GET_MASK_MODE): New. * config/i386/sse.md (avx512fmaskmodelower): New. (vec_cmp): New. (vec_cmp): New. (vec_cmpv2div2di): New. (vec_cmpu): New. (vec_cmpu): New. (vec_cmpuv2div2di): New. gcc/testsuite/ * gcc.dg/vect/slp-cond-5.c: New test. From-SVN: r230098 --- gcc/tree-vect-stmts.c | 229 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 221 insertions(+), 8 deletions(-) (limited to 'gcc/tree-vect-stmts.c') diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 51dff9e..af203ab 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -1300,7 +1300,7 @@ vect_init_vector (gimple *stmt, tree val, tree type, gimple_stmt_iterator *gsi) if (!types_compatible_p (TREE_TYPE (type), TREE_TYPE (val))) { if (CONSTANT_CLASS_P (val)) - val = fold_unary (VIEW_CONVERT_EXPR, TREE_TYPE (type), val); + val = fold_convert (TREE_TYPE (type), val); else { new_temp = make_ssa_name (TREE_TYPE (type)); @@ -1328,16 +1328,18 @@ vect_init_vector (gimple *stmt, tree val, tree type, gimple_stmt_iterator *gsi) STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def. In case OP is an invariant or constant, a new stmt that creates a vector def - needs to be introduced. */ + needs to be introduced. VECTYPE may be used to specify a required type for + vector invariant. */ tree -vect_get_vec_def_for_operand (tree op, gimple *stmt) +vect_get_vec_def_for_operand (tree op, gimple *stmt, tree vectype) { tree vec_oprnd; gimple *vec_stmt; gimple *def_stmt; stmt_vec_info def_stmt_info = NULL; stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt); + tree stmt_vectype = STMT_VINFO_VECTYPE (stmt_vinfo); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo); enum vect_def_type dt; bool is_simple_use; @@ -1372,7 +1374,14 @@ vect_get_vec_def_for_operand (tree op, gimple *stmt) case vect_constant_def: case vect_external_def: { - vector_type = get_vectype_for_scalar_type (TREE_TYPE (op)); + if (vectype) + vector_type = vectype; + else if (TREE_CODE (TREE_TYPE (op)) == BOOLEAN_TYPE + && VECTOR_BOOLEAN_TYPE_P (stmt_vectype)) + vector_type = build_same_sized_truth_vector_type (stmt_vectype); + else + vector_type = get_vectype_for_scalar_type (TREE_TYPE (op)); + gcc_assert (vector_type); return vect_init_vector (stmt, op, vector_type, NULL); } @@ -7329,13 +7338,14 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, { gimple *gtemp; vec_cond_lhs = - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), stmt); + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), + stmt, comp_vectype); vect_is_simple_use (TREE_OPERAND (cond_expr, 0), loop_vinfo, >emp, &dts[0]); vec_cond_rhs = vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), - stmt); + stmt, comp_vectype); vect_is_simple_use (TREE_OPERAND (cond_expr, 1), loop_vinfo, >emp, &dts[1]); if (reduc_index == 1) @@ -7416,6 +7426,185 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, return true; } +/* vectorizable_comparison. + + Check if STMT is comparison expression that can be vectorized. + If VEC_STMT is also passed, vectorize the STMT: create a vectorized + comparison, put it in VEC_STMT, and insert it at GSI. + + Return FALSE if not a vectorizable STMT, TRUE otherwise. */ + +bool +vectorizable_comparison (gimple *stmt, gimple_stmt_iterator *gsi, + gimple **vec_stmt, tree reduc_def, + slp_tree slp_node) +{ + tree lhs, rhs1, rhs2; + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + tree vectype1 = NULL_TREE, vectype2 = NULL_TREE; + tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE; + tree new_temp; + loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); + enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type}; + unsigned nunits; + int ncopies; + enum tree_code code; + stmt_vec_info prev_stmt_info = NULL; + int i, j; + bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); + vec vec_oprnds0 = vNULL; + vec vec_oprnds1 = vNULL; + gimple *def_stmt; + tree mask_type; + tree mask; + + if (!VECTOR_BOOLEAN_TYPE_P (vectype)) + return false; + + mask_type = vectype; + nunits = TYPE_VECTOR_SUBPARTS (vectype); + + if (slp_node || PURE_SLP_STMT (stmt_info)) + ncopies = 1; + else + ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits; + + gcc_assert (ncopies >= 1); + if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo) + return false; + + if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def + && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle + && reduc_def)) + return false; + + if (STMT_VINFO_LIVE_P (stmt_info)) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "value used after loop.\n"); + return false; + } + + if (!is_gimple_assign (stmt)) + return false; + + code = gimple_assign_rhs_code (stmt); + + if (TREE_CODE_CLASS (code) != tcc_comparison) + return false; + + rhs1 = gimple_assign_rhs1 (stmt); + rhs2 = gimple_assign_rhs2 (stmt); + + if (!vect_is_simple_use (rhs1, stmt_info->vinfo, &def_stmt, + &dts[0], &vectype1)) + return false; + + if (!vect_is_simple_use (rhs2, stmt_info->vinfo, &def_stmt, + &dts[1], &vectype2)) + return false; + + if (vectype1 && vectype2 + && TYPE_VECTOR_SUBPARTS (vectype1) != TYPE_VECTOR_SUBPARTS (vectype2)) + return false; + + vectype = vectype1 ? vectype1 : vectype2; + + /* Invariant comparison. */ + if (!vectype) + { + vectype = build_vector_type (TREE_TYPE (rhs1), nunits); + if (tree_to_shwi (TYPE_SIZE_UNIT (vectype)) != current_vector_size) + return false; + } + else if (nunits != TYPE_VECTOR_SUBPARTS (vectype)) + return false; + + if (!vec_stmt) + { + STMT_VINFO_TYPE (stmt_info) = comparison_vec_info_type; + vect_model_simple_cost (stmt_info, ncopies, dts, NULL, NULL); + return expand_vec_cmp_expr_p (vectype, mask_type); + } + + /* Transform. */ + if (!slp_node) + { + vec_oprnds0.create (1); + vec_oprnds1.create (1); + } + + /* Handle def. */ + lhs = gimple_assign_lhs (stmt); + mask = vect_create_destination_var (lhs, mask_type); + + /* Handle cmp expr. */ + for (j = 0; j < ncopies; j++) + { + gassign *new_stmt = NULL; + if (j == 0) + { + if (slp_node) + { + auto_vec ops; + auto_vec, 2> vec_defs; + + ops.safe_push (rhs1); + ops.safe_push (rhs2); + vect_get_slp_defs (ops, slp_node, &vec_defs, -1); + vec_oprnds1 = vec_defs.pop (); + vec_oprnds0 = vec_defs.pop (); + } + else + { + vec_rhs1 = vect_get_vec_def_for_operand (rhs1, stmt, NULL); + vec_rhs2 = vect_get_vec_def_for_operand (rhs2, stmt, NULL); + } + } + else + { + vec_rhs1 = vect_get_vec_def_for_stmt_copy (dts[0], + vec_oprnds0.pop ()); + vec_rhs2 = vect_get_vec_def_for_stmt_copy (dts[1], + vec_oprnds1.pop ()); + } + + if (!slp_node) + { + vec_oprnds0.quick_push (vec_rhs1); + vec_oprnds1.quick_push (vec_rhs2); + } + + /* Arguments are ready. Create the new vector stmt. */ + FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1) + { + vec_rhs2 = vec_oprnds1[i]; + + new_temp = make_ssa_name (mask); + new_stmt = gimple_build_assign (new_temp, code, vec_rhs1, vec_rhs2); + vect_finish_stmt_generation (stmt, new_stmt, gsi); + if (slp_node) + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + } + + if (slp_node) + continue; + + if (j == 0) + STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt; + else + STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt; + + prev_stmt_info = vinfo_for_stmt (new_stmt); + } + + vec_oprnds0.release (); + vec_oprnds1.release (); + + return true; +} /* Make sure the statement is vectorizable. */ @@ -7619,7 +7808,8 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node) || vectorizable_call (stmt, NULL, NULL, node) || vectorizable_store (stmt, NULL, NULL, node) || vectorizable_reduction (stmt, NULL, NULL, node) - || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node)); + || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node) + || vectorizable_comparison (stmt, NULL, NULL, NULL, node)); else { if (bb_vinfo) @@ -7631,7 +7821,8 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node) || vectorizable_load (stmt, NULL, NULL, node, NULL) || vectorizable_call (stmt, NULL, NULL, node) || vectorizable_store (stmt, NULL, NULL, node) - || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node)); + || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node) + || vectorizable_comparison (stmt, NULL, NULL, NULL, node)); } if (!ok) @@ -7747,6 +7938,11 @@ vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi, gcc_assert (done); break; + case comparison_vec_info_type: + done = vectorizable_comparison (stmt, gsi, &vec_stmt, NULL, slp_node); + gcc_assert (done); + break; + case call_vec_info_type: done = vectorizable_call (stmt, gsi, &vec_stmt, slp_node); stmt = gsi_stmt (*gsi); @@ -8080,6 +8276,23 @@ get_vectype_for_scalar_type (tree scalar_type) return vectype; } +/* Function get_mask_type_for_scalar_type. + + Returns the mask type corresponding to a result of comparison + of vectors of specified SCALAR_TYPE as supported by target. */ + +tree +get_mask_type_for_scalar_type (tree scalar_type) +{ + tree vectype = get_vectype_for_scalar_type (scalar_type); + + if (!vectype) + return NULL; + + return build_truth_vector_type (TYPE_VECTOR_SUBPARTS (vectype), + current_vector_size); +} + /* Function get_same_sized_vectype Returns a vector type corresponding to SCALAR_TYPE of size -- cgit v1.1