cleanup: make all cond_len_* and mask_len_* consistent on the order of mask and len

This patch is depending on:
https://gcc.gnu.org/pipermail/gcc-patches/2023-July/625121.html

Hi, Richard and Richi.

This patch is to align the order of mask and len.

Currently, According to this piece code:
			if (final_len && final_mask)
			  call = gimple_build_call_internal (
			    IFN_LEN_MASK_GATHER_LOAD, 7, dataref_ptr,
			    vec_offset, scale, zero, final_mask, final_len,
			    bias);

You can see the order of mask and len, is {mask,len,bias}.
"mask" comes before "len". The reason of this order is that we want to
reuse the current codes of MASK_GATHER_LOAD/MASK_SCATTER_STORE.

Same situation for COND_LEN_*, we want to reuse the codes of COND_*.

Reusing codes from the existing MASK_* or COND_* can allow us not to
change the codes too much and make the codes elegant and easy to maintain && read.

To avoid any confusions of auto-vectorization patterns that includes both mask and len,

this patch align the order of mask and len for both Gimple IR and RTL pattern into

{mask, len, bias} to make everything cleaner and more elegant.

Bootstrap and Regression is on the way.

gcc/ChangeLog:

	* config/riscv/autovec.md: Align order of mask and len.
	* config/riscv/riscv-v.cc (expand_load_store): Ditto.
	(expand_gather_scatter): Ditto.
	* doc/md.texi: Ditto.
	* internal-fn.cc (add_len_and_mask_args): Ditto.
	(add_mask_and_len_args): Ditto.
	(expand_partial_load_optab_fn): Ditto.
	(expand_partial_store_optab_fn): Ditto.
	(expand_scatter_store_optab_fn): Ditto.
	(expand_gather_load_optab_fn): Ditto.
	(internal_fn_len_index): Ditto.
	(internal_fn_mask_index): Ditto.
	(internal_len_load_store_bias): Ditto.
	* tree-vect-stmts.cc (vectorizable_store): Ditto.
	(vectorizable_load): Ditto.

5 files changed, 101 insertions, 101 deletions

diff --git a/gcc/config/riscv/autovec.md b/gcc/config/riscv/autovec.md
index f06aff7..c9ea731 100644
--- a/gcc/config/riscv/autovec.md
+++ b/gcc/config/riscv/autovec.md
@@ -25,9 +25,9 @@
 (define_expand "mask_len_load<mode><vm>"
   [(match_operand:V 0 "register_operand")
    (match_operand:V 1 "memory_operand")
-   (match_operand 2 "autovec_length_operand")
-   (match_operand 3 "const_0_operand")
-   (match_operand:<VM> 4 "vector_mask_operand")]
+   (match_operand:<VM> 2 "vector_mask_operand")
+   (match_operand 3 "autovec_length_operand")
+   (match_operand 4 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_load_store (operands, true);
@@ -37,9 +37,9 @@
 (define_expand "mask_len_store<mode><vm>"
   [(match_operand:V 0 "memory_operand")
    (match_operand:V 1 "register_operand")
-   (match_operand 2 "autovec_length_operand")
-   (match_operand 3 "const_0_operand")
-   (match_operand:<VM> 4 "vector_mask_operand")]
+   (match_operand:<VM> 2 "vector_mask_operand")
+   (match_operand 3 "autovec_length_operand")
+   (match_operand 4 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_load_store (operands, false);
@@ -67,9 +67,9 @@
    (match_operand:RATIO64I 2 "register_operand")
    (match_operand 3 "<RATIO64:gs_extension>")
    (match_operand 4 "<RATIO64:gs_scale>")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO64:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO64:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, true);
@@ -82,9 +82,9 @@
    (match_operand:RATIO32I 2 "register_operand")
    (match_operand 3 "<RATIO32:gs_extension>")
    (match_operand 4 "<RATIO32:gs_scale>")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO32:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO32:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, true);
@@ -97,9 +97,9 @@
    (match_operand:RATIO16I 2 "register_operand")
    (match_operand 3 "<RATIO16:gs_extension>")
    (match_operand 4 "<RATIO16:gs_scale>")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO16:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO16:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, true);
@@ -112,9 +112,9 @@
    (match_operand:RATIO8I 2 "register_operand")
    (match_operand 3 "<RATIO8:gs_extension>")
    (match_operand 4 "<RATIO8:gs_scale>")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO8:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO8:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, true);
@@ -127,9 +127,9 @@
    (match_operand:RATIO4I 2 "register_operand")
    (match_operand 3 "<RATIO4:gs_extension>")
    (match_operand 4 "<RATIO4:gs_scale>")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO4:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO4:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, true);
@@ -142,9 +142,9 @@
    (match_operand:RATIO2I 2 "register_operand")
    (match_operand 3 "<RATIO2:gs_extension>")
    (match_operand 4 "<RATIO2:gs_scale>")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO2:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO2:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, true);
@@ -161,9 +161,9 @@
    (match_operand:RATIO1 2 "register_operand")
    (match_operand 3 "<RATIO1:gs_extension>")
    (match_operand 4 "<RATIO1:gs_scale>")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO1:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO1:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, true);
@@ -180,9 +180,9 @@
    (match_operand 2 "<RATIO64:gs_extension>")
    (match_operand 3 "<RATIO64:gs_scale>")
    (match_operand:RATIO64 4 "register_operand")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO64:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO64:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, false);
@@ -195,9 +195,9 @@
    (match_operand 2 "<RATIO32:gs_extension>")
    (match_operand 3 "<RATIO32:gs_scale>")
    (match_operand:RATIO32 4 "register_operand")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO32:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO32:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, false);
@@ -210,9 +210,9 @@
    (match_operand 2 "<RATIO16:gs_extension>")
    (match_operand 3 "<RATIO16:gs_scale>")
    (match_operand:RATIO16 4 "register_operand")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO16:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO16:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, false);
@@ -225,9 +225,9 @@
    (match_operand 2 "<RATIO8:gs_extension>")
    (match_operand 3 "<RATIO8:gs_scale>")
    (match_operand:RATIO8 4 "register_operand")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO8:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO8:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, false);
@@ -240,9 +240,9 @@
    (match_operand 2 "<RATIO4:gs_extension>")
    (match_operand 3 "<RATIO4:gs_scale>")
    (match_operand:RATIO4 4 "register_operand")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO4:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO4:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, false);
@@ -255,9 +255,9 @@
    (match_operand 2 "<RATIO2:gs_extension>")
    (match_operand 3 "<RATIO2:gs_scale>")
    (match_operand:RATIO2 4 "register_operand")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO2:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO2:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, false);
@@ -274,9 +274,9 @@
    (match_operand 2 "<RATIO1:gs_extension>")
    (match_operand 3 "<RATIO1:gs_scale>")
    (match_operand:RATIO1 4 "register_operand")
-   (match_operand 5 "autovec_length_operand")
-   (match_operand 6 "const_0_operand")
-   (match_operand:<RATIO1:VM> 7 "vector_mask_operand")]
+   (match_operand:<RATIO1:VM> 5 "vector_mask_operand")
+   (match_operand 6 "autovec_length_operand")
+   (match_operand 7 "const_0_operand")]
   "TARGET_VECTOR"
 {
   riscv_vector::expand_gather_scatter (operands, false);
diff --git a/gcc/config/riscv/riscv-v.cc b/gcc/config/riscv/riscv-v.cc
index 53088ed..eb92451 100644
--- a/gcc/config/riscv/riscv-v.cc
+++ b/gcc/config/riscv/riscv-v.cc
@@ -3073,13 +3073,13 @@ expand_select_vl (rtx *ops)
   emit_insn (gen_no_side_effects_vsetvl_rtx (rvv_mode, ops[0], ops[1]));
 }
 
-/* Expand LEN_MASK_{LOAD,STORE}.  */
+/* Expand MASK_LEN_{LOAD,STORE}.  */
 void
 expand_load_store (rtx *ops, bool is_load)
 {
   poly_int64 value;
-  rtx len = ops[2];
-  rtx mask = ops[4];
+  rtx mask = ops[2];
+  rtx len = ops[3];
   machine_mode mode = GET_MODE (ops[0]);
 
   if (poly_int_rtx_p (len, &value) && known_eq (value, GET_MODE_NUNITS (mode)))
@@ -3215,6 +3215,8 @@ expand_gather_scatter (rtx *ops, bool is_load)
   rtx ptr, vec_offset, vec_reg, len, mask;
   bool zero_extend_p;
   int scale_log2;
+  rtx mask = ops[5];
+  rtx len = ops[6];
   if (is_load)
     {
       vec_reg = ops[0];
@@ -3222,8 +3224,6 @@ expand_gather_scatter (rtx *ops, bool is_load)
       vec_offset = ops[2];
       zero_extend_p = INTVAL (ops[3]);
       scale_log2 = exact_log2 (INTVAL (ops[4]));
-      len = ops[5];
-      mask = ops[7];
     }
   else
     {
@@ -3232,8 +3232,6 @@ expand_gather_scatter (rtx *ops, bool is_load)
       vec_offset = ops[1];
       zero_extend_p = INTVAL (ops[2]);
       scale_log2 = exact_log2 (INTVAL (ops[3]));
-      len = ops[5];
-      mask = ops[7];
     }
 
   machine_mode vec_mode = GET_MODE (vec_reg);
diff --git a/gcc/doc/md.texi b/gcc/doc/md.texi
index f864936..9693b6b 100644
--- a/gcc/doc/md.texi
+++ b/gcc/doc/md.texi
@@ -5042,12 +5042,12 @@ of the result should be set to zero.
 
 @cindex @code{mask_len_gather_load@var{m}@var{n}} instruction pattern
 @item @samp{mask_len_gather_load@var{m}@var{n}}
-Like @samp{gather_load@var{m}@var{n}}, but takes an extra length operand (operand 5),
-a bias operand (operand 6) as well as a mask operand (operand 7).  Similar to mask_len_load,
-the instruction loads at most (operand 5 + operand 6) elements from memory.
+Like @samp{gather_load@var{m}@var{n}}, but takes an extra mask operand (operand 5),
+a len operand (operand 6) as well as a bias operand (operand 7).  Similar to mask_len_load,
+the instruction loads at most (operand 6 + operand 7) elements from memory.
 Bit @var{i} of the mask is set if element @var{i} of the result should
 be loaded from memory and clear if element @var{i} of the result should be undefined.
-Mask elements @var{i} with @var{i} > (operand 5 + operand 6) are ignored.
+Mask elements @var{i} with @var{i} > (operand 6 + operand 7) are ignored.
 
 @cindex @code{scatter_store@var{m}@var{n}} instruction pattern
 @item @samp{scatter_store@var{m}@var{n}}
@@ -5080,11 +5080,11 @@ of the result should be stored to memory.
 
 @cindex @code{mask_len_scatter_store@var{m}@var{n}} instruction pattern
 @item @samp{mask_len_scatter_store@var{m}@var{n}}
-Like @samp{scatter_store@var{m}@var{n}}, but takes an extra length operand (operand 5),
-a bias operand (operand 6) as well as a mask operand (operand 7).  The instruction stores
-at most (operand 5 + operand 6) elements of (operand 4) to memory.
+Like @samp{scatter_store@var{m}@var{n}}, but takes an extra mask operand (operand 5),
+a len operand (operand 6) as well as a bias operand (operand 7).  The instruction stores
+at most (operand 6 + operand 7) elements of (operand 4) to memory.
 Bit @var{i} of the mask is set if element @var{i} of (operand 4) should be stored.
-Mask elements @var{i} with @var{i} > (operand 5 + operand 6) are ignored.
+Mask elements @var{i} with @var{i} > (operand 6 + operand 7) are ignored.
 
 @cindex @code{vec_set@var{m}} instruction pattern
 @item @samp{vec_set@var{m}}
@@ -5324,15 +5324,15 @@ This pattern is not allowed to @code{FAIL}.
 @cindex @code{mask_len_load@var{m}@var{n}} instruction pattern
 @item @samp{mask_len_load@var{m}@var{n}}
 Perform a masked load from the memory location pointed to by operand 1
-into register operand 0.  (operand 2 + operand 3) elements are loaded from
+into register operand 0.  (operand 3 + operand 4) elements are loaded from
 memory and other elements in operand 0 are set to undefined values.
 This is a combination of len_load and maskload.
-Operands 0 and 1 have mode @var{m}, which must be a vector mode.  Operand 2
+Operands 0 and 1 have mode @var{m}, which must be a vector mode.  Operand 3
 has whichever integer mode the target prefers.  A mask is specified in
-operand 4 which must be of type @var{n}.  The mask has lower precedence than
+operand 2 which must be of type @var{n}.  The mask has lower precedence than
 the length and is itself subject to length masking,
-i.e. only mask indices < (operand 2 + operand 3) are used.
-Operand 3 conceptually has mode @code{QI}.
+i.e. only mask indices < (operand 3 + operand 4) are used.
+Operand 4 conceptually has mode @code{QI}.
 
 Operand 2 can be a variable or a constant amount.  Operand 4 specifies a
 constant bias: it is either a constant 0 or a constant -1.  The predicate on
@@ -5351,14 +5351,14 @@ This pattern is not allowed to @code{FAIL}.
 @cindex @code{mask_len_store@var{m}@var{n}} instruction pattern
 @item @samp{mask_len_store@var{m}@var{n}}
 Perform a masked store from vector register operand 1 into memory operand 0.
-(operand 2 + operand 3) elements are stored to memory
+(operand 3 + operand 4) elements are stored to memory
 and leave the other elements of operand 0 unchanged.
 This is a combination of len_store and maskstore.
-Operands 0 and 1 have mode @var{m}, which must be a vector mode.  Operand 2 has whichever
-integer mode the target prefers.  A mask is specified in operand 4 which must be
+Operands 0 and 1 have mode @var{m}, which must be a vector mode.  Operand 3 has whichever
+integer mode the target prefers.  A mask is specified in operand 2 which must be
 of type @var{n}.  The mask has lower precedence than the length and is itself subject to
-length masking, i.e. only mask indices < (operand 2 + operand 3) are used.
-Operand 3 conceptually has mode @code{QI}.
+length masking, i.e. only mask indices < (operand 3 + operand 4) are used.
+Operand 4 conceptually has mode @code{QI}.
 
 Operand 2 can be a variable or a constant amount.  Operand 3 specifies a
 constant bias: it is either a constant 0 or a constant -1.  The predicate on
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index ad14960..8e29428 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -298,10 +298,10 @@ get_multi_vector_move (tree array_type, convert_optab optab)
   return convert_optab_handler (optab, imode, vmode);
 }
 
-/* Add len and mask arguments according to the STMT.  */
+/* Add mask and len arguments according to the STMT.  */
 
 static unsigned int
-add_len_and_mask_args (expand_operand *ops, unsigned int opno, gcall *stmt)
+add_mask_and_len_args (expand_operand *ops, unsigned int opno, gcall *stmt)
 {
   internal_fn ifn = gimple_call_internal_fn (stmt);
   int len_index = internal_fn_len_index (ifn);
@@ -309,6 +309,13 @@ add_len_and_mask_args (expand_operand *ops, unsigned int opno, gcall *stmt)
   int bias_index = len_index + 1;
   int mask_index = internal_fn_mask_index (ifn);
   /* The order of arguments are always {len,bias,mask}.  */
+  if (mask_index >= 0)
+    {
+      tree mask = gimple_call_arg (stmt, mask_index);
+      rtx mask_rtx = expand_normal (mask);
+      create_input_operand (&ops[opno++], mask_rtx,
+			    TYPE_MODE (TREE_TYPE (mask)));
+    }
   if (len_index >= 0)
     {
       tree len = gimple_call_arg (stmt, len_index);
@@ -320,13 +327,6 @@ add_len_and_mask_args (expand_operand *ops, unsigned int opno, gcall *stmt)
       rtx bias = expand_normal (biast);
       create_input_operand (&ops[opno++], bias, QImode);
     }
-  if (mask_index >= 0)
-    {
-      tree mask = gimple_call_arg (stmt, mask_index);
-      rtx mask_rtx = expand_normal (mask);
-      create_input_operand (&ops[opno++], mask_rtx,
-			    TYPE_MODE (TREE_TYPE (mask)));
-    }
   return opno;
 }
 
@@ -2944,7 +2944,7 @@ expand_partial_load_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
   target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
   create_output_operand (&ops[i++], target, TYPE_MODE (type));
   create_fixed_operand (&ops[i++], mem);
-  i = add_len_and_mask_args (ops, i, stmt);
+  i = add_mask_and_len_args (ops, i, stmt);
   expand_insn (icode, i, ops);
 
   if (!rtx_equal_p (target, ops[0].value))
@@ -2986,7 +2986,7 @@ expand_partial_store_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab
   reg = expand_normal (rhs);
   create_fixed_operand (&ops[i++], mem);
   create_input_operand (&ops[i++], reg, TYPE_MODE (type));
-  i = add_len_and_mask_args (ops, i, stmt);
+  i = add_mask_and_len_args (ops, i, stmt);
   expand_insn (icode, i, ops);
 }
 
@@ -3579,7 +3579,7 @@ expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
   create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
   create_integer_operand (&ops[i++], scale_int);
   create_input_operand (&ops[i++], rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
-  i = add_len_and_mask_args (ops, i, stmt);
+  i = add_mask_and_len_args (ops, i, stmt);
 
   insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)),
 					   TYPE_MODE (TREE_TYPE (offset)));
@@ -3608,7 +3608,7 @@ expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
   create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
   create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
   create_integer_operand (&ops[i++], scale_int);
-  i = add_len_and_mask_args (ops, i, stmt);
+  i = add_mask_and_len_args (ops, i, stmt);
   insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)),
 					   TYPE_MODE (TREE_TYPE (offset)));
   expand_insn (icode, i, ops);
@@ -4616,8 +4616,6 @@ internal_fn_len_index (internal_fn fn)
     {
     case IFN_LEN_LOAD:
     case IFN_LEN_STORE:
-    case IFN_MASK_LEN_LOAD:
-    case IFN_MASK_LEN_STORE:
       return 2;
 
     case IFN_MASK_LEN_GATHER_LOAD:
@@ -4646,6 +4644,8 @@ internal_fn_len_index (internal_fn fn)
       return 4;
 
     case IFN_COND_LEN_NEG:
+    case IFN_MASK_LEN_LOAD:
+    case IFN_MASK_LEN_STORE:
       return 3;
 
     default:
@@ -4665,12 +4665,12 @@ internal_fn_mask_index (internal_fn fn)
     case IFN_MASK_LOAD_LANES:
     case IFN_MASK_STORE:
     case IFN_MASK_STORE_LANES:
+    case IFN_MASK_LEN_LOAD:
+    case IFN_MASK_LEN_STORE:
       return 2;
 
     case IFN_MASK_GATHER_LOAD:
     case IFN_MASK_SCATTER_STORE:
-    case IFN_MASK_LEN_LOAD:
-    case IFN_MASK_LEN_STORE:
     case IFN_MASK_LEN_GATHER_LOAD:
     case IFN_MASK_LEN_SCATTER_STORE:
       return 4;
@@ -4755,17 +4755,18 @@ internal_check_ptrs_fn_supported_p (internal_fn ifn, tree type,
 	  && insn_operand_matches (icode, 4, GEN_INT (align)));
 }
 
-/* Return the supported bias for IFN which is either IFN_LEN_LOAD
-   or IFN_LEN_STORE.  For now we only support the biases of 0 and -1
-   (in case 0 is not an allowable length for len_load or len_store).
-   If none of the biases match what the backend provides, return
-   VECT_PARTIAL_BIAS_UNSUPPORTED.  */
+/* Return the supported bias for IFN which is either IFN_{LEN_,MASK_LEN_,}LOAD
+   or IFN_{LEN_,MASK_LEN_,}STORE.  For now we only support the biases of 0 and
+   -1 (in case 0 is not an allowable length for {len_,mask_len_}load or
+   {len_,mask_len_}store). If none of the biases match what the backend
+   provides, return VECT_PARTIAL_BIAS_UNSUPPORTED.  */
 
 signed char
 internal_len_load_store_bias (internal_fn ifn, machine_mode mode)
 {
   optab optab = direct_internal_fn_optab (ifn);
   insn_code icode = direct_optab_handler (optab, mode);
+  int bias_no = 3;
 
   if (icode == CODE_FOR_nothing)
     {
@@ -4783,14 +4784,15 @@ internal_len_load_store_bias (internal_fn ifn, machine_mode mode)
 	  optab = direct_internal_fn_optab (IFN_MASK_LEN_STORE);
 	}
       icode = convert_optab_handler (optab, mode, mask_mode);
+      bias_no = 4;
     }
 
   if (icode != CODE_FOR_nothing)
     {
       /* For now we only support biases of 0 or -1.  Try both of them.  */
-      if (insn_operand_matches (icode, 3, GEN_INT (0)))
+      if (insn_operand_matches (icode, bias_no, GEN_INT (0)))
 	return 0;
-      if (insn_operand_matches (icode, 3, GEN_INT (-1)))
+      if (insn_operand_matches (icode, bias_no, GEN_INT (-1)))
 	return -1;
     }
 
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index cebddd4..2555958 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -9089,8 +9089,8 @@ vectorizable_store (vec_info *vinfo,
 		  if (partial_ifn == IFN_MASK_LEN_STORE)
 		    call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
 						       dataref_ptr, ptr,
-						       final_len, bias,
-						       final_mask, vec_oprnd);
+						       final_mask, final_len,
+						       bias, vec_oprnd);
 		  else
 		    call
 		      = gimple_build_call_internal (IFN_LEN_STORE, 5,
@@ -10656,8 +10656,8 @@ vectorizable_load (vec_info *vinfo,
 			if (partial_ifn == IFN_MASK_LEN_LOAD)
 			  call = gimple_build_call_internal (IFN_MASK_LEN_LOAD,
 							     5, dataref_ptr,
-							     ptr, final_len,
-							     bias, final_mask);
+							     ptr, final_mask,
+							     final_len, bias);
 			else
 			  call = gimple_build_call_internal (IFN_LEN_LOAD, 4,
 							     dataref_ptr, ptr,