diff options
Diffstat (limited to 'gcc/gimple-fold.cc')
| -rw-r--r-- | gcc/gimple-fold.cc | 29 |
1 files changed, 22 insertions, 7 deletions
diff --git a/gcc/gimple-fold.cc b/gcc/gimple-fold.cc index 55e8056..6d167b1 100644 --- a/gcc/gimple-fold.cc +++ b/gcc/gimple-fold.cc @@ -5370,10 +5370,10 @@ arith_overflowed_p (enum tree_code code, const_tree type, return wi::min_precision (wres, sign) > TYPE_PRECISION (type); } -/* If IFN_{MASK,LEN}_LOAD/STORE call CALL is unconditional, return a MEM_REF - for the memory it references, otherwise return null. VECTYPE is the - type of the memory vector. MASK_P indicates it's for MASK if true, - otherwise it's for LEN. */ +/* If IFN_{MASK,LEN,LEN_MASK}_LOAD/STORE call CALL is unconditional, + return a MEM_REF for the memory it references, otherwise return null. + VECTYPE is the type of the memory vector. MASK_P indicates it's for + MASK if true, otherwise it's for LEN. */ static tree gimple_fold_partial_load_store_mem_ref (gcall *call, tree vectype, bool mask_p) @@ -5397,9 +5397,21 @@ gimple_fold_partial_load_store_mem_ref (gcall *call, tree vectype, bool mask_p) unsigned int nargs = gimple_call_num_args (call); tree bias = gimple_call_arg (call, nargs - 1); gcc_assert (TREE_CODE (bias) == INTEGER_CST); - if (maybe_ne (wi::to_poly_widest (basic_len) - wi::to_widest (bias), - GET_MODE_SIZE (TYPE_MODE (vectype)))) + /* For LEN_LOAD/LEN_STORE/LEN_MASK_LOAD/LEN_MASK_STORE, + we don't fold when (bias + len) != VF. */ + if (maybe_ne (wi::to_poly_widest (basic_len) + wi::to_widest (bias), + GET_MODE_NUNITS (TYPE_MODE (vectype)))) return NULL_TREE; + + /* For LEN_MASK_{LOAD,STORE}, we should also check whether + the mask is all ones mask. */ + internal_fn ifn = gimple_call_internal_fn (call); + if (ifn == IFN_LEN_MASK_LOAD || ifn == IFN_LEN_MASK_STORE) + { + tree mask = gimple_call_arg (call, internal_fn_mask_index (ifn)); + if (!integer_all_onesp (mask)) + return NULL_TREE; + } } unsigned HOST_WIDE_INT align = tree_to_uhwi (alias_align); @@ -5438,7 +5450,8 @@ static bool gimple_fold_partial_store (gimple_stmt_iterator *gsi, gcall *call, bool mask_p) { - tree rhs = gimple_call_arg (call, 3); + internal_fn ifn = gimple_call_internal_fn (call); + tree rhs = gimple_call_arg (call, internal_fn_stored_value_index (ifn)); if (tree lhs = gimple_fold_partial_load_store_mem_ref (call, TREE_TYPE (rhs), mask_p)) { @@ -5676,9 +5689,11 @@ gimple_fold_call (gimple_stmt_iterator *gsi, bool inplace) changed |= gimple_fold_partial_store (gsi, stmt, true); break; case IFN_LEN_LOAD: + case IFN_LEN_MASK_LOAD: changed |= gimple_fold_partial_load (gsi, stmt, false); break; case IFN_LEN_STORE: + case IFN_LEN_MASK_STORE: changed |= gimple_fold_partial_store (gsi, stmt, false); break; default: |