From 8af3f53d325fe4a6b1f3da64dc02d10978b9a116 Mon Sep 17 00:00:00 2001 From: Richard Sandiford Date: Tue, 30 Nov 2021 09:52:30 +0000 Subject: vect: Support masked gather loads with SLP This patch extends the previous SLP gather load support so that it can handle masked loads too. gcc/ * tree-vect-slp.c (arg1_arg4_map): New variable. (vect_get_operand_map): Handle IFN_MASK_GATHER_LOAD. (vect_build_slp_tree_1): Likewise. (vect_build_slp_tree_2): Likewise. * tree-vect-stmts.c (vectorizable_load): Expect the mask to be the last SLP child node rather than the first. gcc/testsuite/ * gcc.dg/vect/vect-gather-3.c: New test. * gcc.dg/vect/vect-gather-4.c: Likewise. * gcc.target/aarch64/sve/mask_gather_load_8.c: Likewise. --- gcc/tree-vect-stmts.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'gcc/tree-vect-stmts.c') diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 06da5a9..8642acb 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -8595,6 +8595,7 @@ vectorizable_load (vec_info *vinfo, return false; tree mask = NULL_TREE, mask_vectype = NULL_TREE; + int mask_index = -1; if (gassign *assign = dyn_cast (stmt_info->stmt)) { scalar_dest = gimple_assign_lhs (assign); @@ -8626,12 +8627,12 @@ vectorizable_load (vec_info *vinfo, if (!scalar_dest) return false; - int mask_index = internal_fn_mask_index (ifn); + mask_index = internal_fn_mask_index (ifn); + /* ??? For SLP the mask operand is always last. */ + if (mask_index >= 0 && slp_node) + mask_index = SLP_TREE_CHILDREN (slp_node).length () - 1; if (mask_index >= 0 - && !vect_check_scalar_mask (vinfo, stmt_info, slp_node, - /* ??? For SLP we only have operands for - the mask operand. */ - slp_node ? 0 : mask_index, + && !vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_index, &mask, NULL, &mask_dt, &mask_vectype)) return false; } @@ -9393,8 +9394,14 @@ vectorizable_load (vec_info *vinfo, vec vec_offsets = vNULL; auto_vec vec_masks; if (mask) - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, - mask, &vec_masks, mask_vectype, NULL_TREE); + { + if (slp_node) + vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[mask_index], + &vec_masks); + else + vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies, mask, + &vec_masks, mask_vectype); + } tree vec_mask = NULL_TREE; poly_uint64 group_elt = 0; for (j = 0; j < ncopies; j++) -- cgit v1.1