From 1694907238eb106bf7ac0e4eaedaa77bc7719b6d Mon Sep 17 00:00:00 2001 From: Richard Guenther Date: Thu, 4 Nov 2010 10:56:22 +0000 Subject: tree.def (FMA_EXPR): New tree code. 2010-11-04 Richard Guenther Richard Henderson * tree.def (FMA_EXPR): New tree code. * expr.c (expand_expr_real_2): Add FMA_EXPR expansion code. * gimple.c (gimple_rhs_class_table): FMA_EXPR is a GIMPLE_TERNARY_RHS. * tree-cfg.c (verify_gimple_assign_ternary): Verify FMA_EXPR types. * tree-inline.c (estimate_operator_cost): Handle FMA_EXPR. * gimple-pretty-print.c (dump_ternary_rhs): Likewise. * tree-ssa-math-opts.c (convert_mult_to_fma): New function. (execute_optimize_widening_mul): Call it. Reorganize to allow dead stmt removal. Move TODO flags ... (pass_optimize_widening_mul): ... here. * flag-types.h (enum fp_contract_mode): New enum. * common.opt (flag_fp_contract_mode): New variable. (-ffp-contract): New option. * opts.c (common_handle_option): Handle it. * doc/invoke.texi (-ffp-contract): Document. * tree.h (fold_fma): Declare. * builtins.c (fold_fma): New function. (fold_builtin_fma): Likewise. (fold_builtin_3): Call it for fma. * fold-const.c (fold_ternary_loc): Fold FMA_EXPR. * optabs.c (optab_for_tree_code): Handle FMA_EXPR. * config/i386/sse.md (fms4, fnma, fnms4): New expanders. * doc/md.texi (fms4, fnma, fnms4): Document new named patterns. * genopinit.c (optabs): Initialize fms_optab, fnma_optab and fnms_optab. * optabs.h (enum optab_index): Add OTI_fms, OTI_fnma and OTI_fnms. (fms_optab, fnma_optab, fnms_optab): New defines. * gimplify.c (gimplify_expr): Handle binary truth expressions explicitly. Handle FMA_EXPR. * tree-vect-stmts.c (vectorizable_operation): Handle ternary operations. * gcc.target/i386/fma4-vector-2.c: New testcase. Co-Authored-By: Richard Henderson From-SVN: r166304 --- gcc/tree-vect-stmts.c | 66 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 15 deletions(-) (limited to 'gcc/tree-vect-stmts.c') diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 4961ccb..2dbc035 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -2343,7 +2343,8 @@ vectorizable_shift (gimple stmt, gimple_stmt_iterator *gsi, /* Function vectorizable_operation. - Check if STMT performs a binary or unary operation that can be vectorized. + Check if STMT performs a binary, unary or ternary operation that can + be vectorized. If VEC_STMT is also passed, vectorize the STMT: create a vectorized stmt to replace it, put it in VEC_STMT, and insert it at BSI. Return FALSE if not a vectorizable STMT, TRUE otherwise. */ @@ -2354,7 +2355,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, { tree vec_dest; tree scalar_dest; - tree op0, op1 = NULL; + tree op0, op1 = NULL_TREE, op2 = NULL_TREE; stmt_vec_info stmt_info = vinfo_for_stmt (stmt); tree vectype; loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); @@ -2366,7 +2367,8 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, int icode; tree def; gimple def_stmt; - enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type}; + enum vect_def_type dt[3] + = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type}; gimple new_stmt = NULL; stmt_vec_info prev_stmt_info; int nunits_in; @@ -2374,8 +2376,8 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, tree vectype_out; int ncopies; int j, i; - VEC(tree,heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL; - tree vop0, vop1; + VEC(tree,heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL, *vec_oprnds2 = NULL; + tree vop0, vop1, vop2; bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); int vf; @@ -2401,10 +2403,11 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, /* Support only unary or binary operations. */ op_type = TREE_CODE_LENGTH (code); - if (op_type != unary_op && op_type != binary_op) + if (op_type != unary_op && op_type != binary_op && op_type != ternary_op) { if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "num. args = %d (not unary/binary op).", op_type); + fprintf (vect_dump, "num. args = %d (not unary/binary/ternary op).", + op_type); return false; } @@ -2441,7 +2444,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, if (nunits_out != nunits_in) return false; - if (op_type == binary_op) + if (op_type == binary_op || op_type == ternary_op) { op1 = gimple_assign_rhs2 (stmt); if (!vect_is_simple_use (op1, loop_vinfo, bb_vinfo, &def_stmt, &def, @@ -2452,6 +2455,17 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, return false; } } + if (op_type == ternary_op) + { + op2 = gimple_assign_rhs3 (stmt); + if (!vect_is_simple_use (op2, loop_vinfo, bb_vinfo, &def_stmt, &def, + &dt[2])) + { + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "use not simple."); + return false; + } + } if (loop_vinfo) vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); @@ -2473,7 +2487,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, || code == RROTATE_EXPR) return false; - optab = optab_for_tree_code (code, vectype, optab_default); + optab = optab_for_tree_code (code, vectype, optab_default); /* Supportable by target? */ if (!optab) @@ -2534,8 +2548,10 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, if (!slp_node) { vec_oprnds0 = VEC_alloc (tree, heap, 1); - if (op_type == binary_op) + if (op_type == binary_op || op_type == ternary_op) vec_oprnds1 = VEC_alloc (tree, heap, 1); + if (op_type == ternary_op) + vec_oprnds2 = VEC_alloc (tree, heap, 1); } /* In case the vectorization factor (VF) is bigger than the number @@ -2597,22 +2613,40 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, /* Handle uses. */ if (j == 0) { - if (op_type == binary_op) + if (op_type == binary_op || op_type == ternary_op) vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1, slp_node); else vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL, slp_node); + if (op_type == ternary_op) + { + vec_oprnds2 = VEC_alloc (tree, heap, 1); + VEC_quick_push (tree, vec_oprnds2, + vect_get_vec_def_for_operand (op2, stmt, NULL)); + } } else - vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1); + { + vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1); + if (op_type == ternary_op) + { + tree vec_oprnd = VEC_pop (tree, vec_oprnds2); + VEC_quick_push (tree, vec_oprnds2, + vect_get_vec_def_for_stmt_copy (dt[2], + vec_oprnd)); + } + } /* Arguments are ready. Create the new vector stmt. */ FOR_EACH_VEC_ELT (tree, vec_oprnds0, i, vop0) { - vop1 = ((op_type == binary_op) - ? VEC_index (tree, vec_oprnds1, i) : NULL); - new_stmt = gimple_build_assign_with_ops (code, vec_dest, vop0, vop1); + vop1 = ((op_type == binary_op || op_type == ternary_op) + ? VEC_index (tree, vec_oprnds1, i) : NULL_TREE); + vop2 = ((op_type == ternary_op) + ? VEC_index (tree, vec_oprnds2, i) : NULL_TREE); + new_stmt = gimple_build_assign_with_ops3 (code, vec_dest, + vop0, vop1, vop2); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); vect_finish_stmt_generation (stmt, new_stmt, gsi); @@ -2633,6 +2667,8 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, VEC_free (tree, heap, vec_oprnds0); if (vec_oprnds1) VEC_free (tree, heap, vec_oprnds1); + if (vec_oprnds2) + VEC_free (tree, heap, vec_oprnds2); return true; } -- cgit v1.1