diff options
Diffstat (limited to 'gcc')
| -rw-r--r-- | gcc/tree-vect-loop.cc | 7 | ||||
| -rw-r--r-- | gcc/tree-vect-stmts.cc | 52 |
2 files changed, 50 insertions, 9 deletions
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index 1cd6c29..ebee803 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -7494,8 +7494,11 @@ vectorizable_reduction (loop_vec_info loop_vinfo, } if (reduc_chain_length == 1 - && direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST, - vectype_in, OPTIMIZE_FOR_SPEED)) + && (direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST, vectype_in, + OPTIMIZE_FOR_SPEED) + || direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST, + vectype_in, + OPTIMIZE_FOR_SPEED))) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc index 413a887..be9f3a2 100644 --- a/gcc/tree-vect-stmts.cc +++ b/gcc/tree-vect-stmts.cc @@ -11740,8 +11740,17 @@ vectorizable_condition (vec_info *vinfo, && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)) { if (reduction_type == EXTRACT_LAST_REDUCTION) - vect_record_loop_mask (loop_vinfo, &LOOP_VINFO_MASKS (loop_vinfo), - ncopies * vec_num, vectype, NULL); + { + if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST, + vectype, OPTIMIZE_FOR_SPEED)) + vect_record_loop_len (loop_vinfo, + &LOOP_VINFO_LENS (loop_vinfo), + ncopies * vec_num, vectype, 1); + else + vect_record_loop_mask (loop_vinfo, + &LOOP_VINFO_MASKS (loop_vinfo), + ncopies * vec_num, vectype, NULL); + } /* Extra inactive lanes should be safe for vect_nested_cycle. */ else if (STMT_VINFO_DEF_TYPE (reduc_info) != vect_nested_cycle) { @@ -11772,7 +11781,13 @@ vectorizable_condition (vec_info *vinfo, mask to the condition, or to its inverse. */ vec_loop_masks *masks = NULL; - if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) + vec_loop_lens *lens = NULL; + if (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)) + { + if (reduction_type == EXTRACT_LAST_REDUCTION) + lens = &LOOP_VINFO_LENS (loop_vinfo); + } + else if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) { if (reduction_type == EXTRACT_LAST_REDUCTION) masks = &LOOP_VINFO_MASKS (loop_vinfo); @@ -11910,7 +11925,8 @@ vectorizable_condition (vec_info *vinfo, /* Force vec_compare to be an SSA_NAME rather than a comparison, in cases where that's necessary. */ - if (masks || reduction_type == EXTRACT_LAST_REDUCTION) + tree len = NULL_TREE, bias = NULL_TREE; + if (masks || lens || reduction_type == EXTRACT_LAST_REDUCTION) { if (!is_gimple_val (vec_compare)) { @@ -11931,6 +11947,23 @@ vectorizable_condition (vec_info *vinfo, vec_compare = vec_compare_name; } + if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST, + vectype, OPTIMIZE_FOR_SPEED)) + { + if (lens) + { + len = vect_get_loop_len (loop_vinfo, gsi, lens, + vec_num * ncopies, vectype, i, 1); + signed char biasval + = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); + bias = build_int_cst (intQI_type_node, biasval); + } + else + { + len = size_int (TYPE_VECTOR_SUBPARTS (vectype)); + bias = build_int_cst (intQI_type_node, 0); + } + } if (masks) { tree loop_mask @@ -11950,9 +11983,14 @@ vectorizable_condition (vec_info *vinfo, { gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt; tree lhs = gimple_get_lhs (old_stmt); - new_stmt = gimple_build_call_internal - (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare, - vec_then_clause); + if (len) + new_stmt = gimple_build_call_internal + (IFN_LEN_FOLD_EXTRACT_LAST, 5, else_clause, vec_compare, + vec_then_clause, len, bias); + else + new_stmt = gimple_build_call_internal + (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare, + vec_then_clause); gimple_call_set_lhs (new_stmt, lhs); SSA_NAME_DEF_STMT (lhs) = new_stmt; if (old_stmt == gsi_stmt (*gsi)) |