/* Extra machine modes for RISC-V target.
   Copyright (C) 2011-2024 Free Software Foundation, Inc.
   Contributed by Andrew Waterman (andrew@sifive.com).
   Based on MIPS target for GNU compiler.

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3, or (at your option)
any later version.

GCC is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.  */

FLOAT_MODE (HF, 2, ieee_half_format);
FLOAT_MODE (TF, 16, ieee_quad_format);
FLOAT_MODE (BF, 2, 0);
/* Reuse definition from arm.  */
ADJUST_FLOAT_FORMAT (BF, &arm_bfloat_half_format);

/* Vector modes.  */

/* Encode the ratio of SEW/LMUL into the mask types. There are the following
 * mask types.  */

/* Encode the ratio of SEW/LMUL into the mask types.
   There are the following mask types.

  n = SEW/LMUL

  |Modes| n = 1 | n = 2 | n = 4 | n = 8 | n = 16 | n = 32 | n = 64 |
  |BI   |RVVM1BI |RVVMF2BI |RVVMF4BI |RVVMF8BI |RVVMF16BI |RVVMF32BI |RVVMF64BI |  */

/* For RVV modes, each boolean value occupies 1-bit.
   4th argument specifies the minimal possible size of the vector mode,
   and will adjust to the right size by ADJUST_BYTESIZE.  */
VECTOR_BOOL_MODE (RVVM1BI, 64, BI, 8);
VECTOR_BOOL_MODE (RVVMF2BI, 32, BI, 4);
VECTOR_BOOL_MODE (RVVMF4BI, 16, BI, 2);
VECTOR_BOOL_MODE (RVVMF8BI, 8, BI, 1);
VECTOR_BOOL_MODE (RVVMF16BI, 4, BI, 1);
VECTOR_BOOL_MODE (RVVMF32BI, 2, BI, 1);
VECTOR_BOOL_MODE (RVVMF64BI, 1, BI, 1);

ADJUST_NUNITS (RVVM1BI, riscv_v_adjust_nunits (RVVM1BImode, 64));
ADJUST_NUNITS (RVVMF2BI, riscv_v_adjust_nunits (RVVMF2BImode, 32));
ADJUST_NUNITS (RVVMF4BI, riscv_v_adjust_nunits (RVVMF4BImode, 16));
ADJUST_NUNITS (RVVMF8BI, riscv_v_adjust_nunits (RVVMF8BImode, 8));
ADJUST_NUNITS (RVVMF16BI, riscv_v_adjust_nunits (RVVMF16BImode, 4));
ADJUST_NUNITS (RVVMF32BI, riscv_v_adjust_nunits (RVVMF32BImode, 2));
ADJUST_NUNITS (RVVMF64BI, riscv_v_adjust_nunits (RVVMF64BImode, 1));

ADJUST_ALIGNMENT (RVVM1BI, 1);
ADJUST_ALIGNMENT (RVVMF2BI, 1);
ADJUST_ALIGNMENT (RVVMF4BI, 1);
ADJUST_ALIGNMENT (RVVMF8BI, 1);
ADJUST_ALIGNMENT (RVVMF16BI, 1);
ADJUST_ALIGNMENT (RVVMF32BI, 1);
ADJUST_ALIGNMENT (RVVMF64BI, 1);

ADJUST_PRECISION (RVVM1BI, riscv_v_adjust_precision (RVVM1BImode, 64));
ADJUST_PRECISION (RVVMF2BI, riscv_v_adjust_precision (RVVMF2BImode, 32));
ADJUST_PRECISION (RVVMF4BI, riscv_v_adjust_precision (RVVMF4BImode, 16));
ADJUST_PRECISION (RVVMF8BI, riscv_v_adjust_precision (RVVMF8BImode, 8));
ADJUST_PRECISION (RVVMF16BI, riscv_v_adjust_precision (RVVMF16BImode, 4));
ADJUST_PRECISION (RVVMF32BI, riscv_v_adjust_precision (RVVMF32BImode, 2));
ADJUST_PRECISION (RVVMF64BI, riscv_v_adjust_precision (RVVMF64BImode, 1));

ADJUST_BYTESIZE (RVVM1BI, riscv_v_adjust_bytesize (RVVM1BImode, 8));
ADJUST_BYTESIZE (RVVMF2BI, riscv_v_adjust_bytesize (RVVMF2BImode, 4));
ADJUST_BYTESIZE (RVVMF4BI, riscv_v_adjust_bytesize (RVVMF4BImode, 2));
ADJUST_BYTESIZE (RVVMF8BI, riscv_v_adjust_bytesize (RVVMF8BImode, 1));
ADJUST_BYTESIZE (RVVMF16BI, riscv_v_adjust_bytesize (RVVMF16BImode, 1));
ADJUST_BYTESIZE (RVVMF32BI, riscv_v_adjust_bytesize (RVVMF32BImode, 1));
ADJUST_BYTESIZE (RVVMF64BI, riscv_v_adjust_bytesize (RVVMF64BImode, 1));

/* Encode SEW and LMUL into data types.
   We enforce the constraint LMUL ≥ SEW/ELEN in the implementation.
   There are the following data types for ELEN = 64.

   |Modes|LMUL=1 |LMUL=2 |LMUL=4 |LMUL=8 |LMUL=1/2|LMUL=1/4|LMUL=1/8|
   |DI   |RVVM1DI|RVVM2DI|RVVM4DI|RVVM8DI|N/A     |N/A     |N/A     |
   |SI   |RVVM1SI|RVVM2SI|RVVM4SI|RVVM8SI|RVVMF2SI|N/A     |N/A     |
   |HI   |RVVM1HI|RVVM2HI|RVVM4HI|RVVM8HI|RVVMF2HI|RVVMF4HI|N/A     |
   |QI   |RVVM1QI|RVVM2QI|RVVM4QI|RVVM8QI|RVVMF2QI|RVVMF4QI|RVVMF8QI|
   |DF   |RVVM1DF|RVVM2DF|RVVM4DF|RVVM8DF|N/A     |N/A     |N/A     |
   |SF   |RVVM1SF|RVVM2SF|RVVM4SF|RVVM8SF|RVVMF2SF|N/A     |N/A     |
   |HF   |RVVM1HF|RVVM2HF|RVVM4HF|RVVM8HF|RVVMF2HF|RVVMF4HF|N/A     |
   |BF   |RVVM1BF|RVVM2BF|RVVM4BF|RVVM8BF|RVVMF2BF|RVVMF4BF|N/A     |

There are the following data types for ELEN = 32.

   |Modes|LMUL=1 |LMUL=2 |LMUL=4 |LMUL=8 |LMUL=1/2|LMUL=1/4|LMUL=1/8|
   |SI   |RVVM1SI|RVVM2SI|RVVM4SI|RVVM8SI|N/A     |N/A     |N/A     |
   |HI   |RVVM1HI|RVVM2HI|RVVM4HI|RVVM8HI|RVVMF2HI|N/A     |N/A     |
   |QI   |RVVM1QI|RVVM2QI|RVVM4QI|RVVM8QI|RVVMF2QI|RVVMF4QI|N/A     |
   |SF   |RVVM1SF|RVVM2SF|RVVM4SF|RVVM8SF|N/A     |N/A     |N/A     |
   |HF   |RVVM1HF|RVVM2HF|RVVM4HF|RVVM8HF|RVVMF2HF|N/A     |N/A     |
   |BF   |RVVM1BF|RVVM2BF|RVVM4BF|RVVM8BF|RVVMF2BF|N/A     |N/A     |  */

#define RVV_WHOLE_MODES(LMUL)                                                  \
  VECTOR_MODE_WITH_PREFIX (RVVM, INT, QI, LMUL, 0);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM, INT, HI, LMUL, 0);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM, FLOAT, BF, LMUL, 0);				\
  VECTOR_MODE_WITH_PREFIX (RVVM, FLOAT, HF, LMUL, 0);                          \
  VECTOR_MODE_WITH_PREFIX (RVVM, INT, SI, LMUL, 0);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM, FLOAT, SF, LMUL, 0);                          \
  VECTOR_MODE_WITH_PREFIX (RVVM, INT, DI, LMUL, 0);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM, FLOAT, DF, LMUL, 0);                          \
                                                                               \
  ADJUST_NUNITS (RVVM##LMUL##QI,                                               \
		 riscv_v_adjust_nunits (RVVM##LMUL##QImode, false, LMUL, 1));  \
  ADJUST_NUNITS (RVVM##LMUL##HI,                                               \
		 riscv_v_adjust_nunits (RVVM##LMUL##HImode, false, LMUL, 1));  \
  ADJUST_NUNITS (RVVM##LMUL##SI,                                               \
		 riscv_v_adjust_nunits (RVVM##LMUL##SImode, false, LMUL, 1));  \
  ADJUST_NUNITS (RVVM##LMUL##DI,                                               \
		 riscv_v_adjust_nunits (RVVM##LMUL##DImode, false, LMUL, 1));  \
  ADJUST_NUNITS (RVVM##LMUL##BF,						\
		 riscv_v_adjust_nunits (RVVM##LMUL##BFmode, false, LMUL, 1));  \
  ADJUST_NUNITS (RVVM##LMUL##HF,                                               \
		 riscv_v_adjust_nunits (RVVM##LMUL##HFmode, false, LMUL, 1));  \
  ADJUST_NUNITS (RVVM##LMUL##SF,                                               \
		 riscv_v_adjust_nunits (RVVM##LMUL##SFmode, false, LMUL, 1));  \
  ADJUST_NUNITS (RVVM##LMUL##DF,                                               \
		 riscv_v_adjust_nunits (RVVM##LMUL##DFmode, false, LMUL, 1));  \
                                                                               \
  ADJUST_ALIGNMENT (RVVM##LMUL##QI, 1);                                        \
  ADJUST_ALIGNMENT (RVVM##LMUL##HI, 2);                                        \
  ADJUST_ALIGNMENT (RVVM##LMUL##SI, 4);                                        \
  ADJUST_ALIGNMENT (RVVM##LMUL##DI, 8);                                        \
  ADJUST_ALIGNMENT (RVVM##LMUL##BF, 2);					\
  ADJUST_ALIGNMENT (RVVM##LMUL##HF, 2);                                        \
  ADJUST_ALIGNMENT (RVVM##LMUL##SF, 4);                                        \
  ADJUST_ALIGNMENT (RVVM##LMUL##DF, 8);

RVV_WHOLE_MODES (1)
RVV_WHOLE_MODES (2)
RVV_WHOLE_MODES (4)
RVV_WHOLE_MODES (8)

#define RVV_FRACT_MODE(TYPE, MODE, LMUL, ALIGN)                                \
  VECTOR_MODE_WITH_PREFIX (RVVMF, TYPE, MODE, LMUL, 0);                        \
  ADJUST_NUNITS (RVVMF##LMUL##MODE,                                            \
		 riscv_v_adjust_nunits (RVVMF##LMUL##MODE##mode, true, LMUL,   \
					1));                                   \
                                                                               \
  ADJUST_ALIGNMENT (RVVMF##LMUL##MODE, ALIGN);

RVV_FRACT_MODE (INT, QI, 2, 1)
RVV_FRACT_MODE (INT, QI, 4, 1)
RVV_FRACT_MODE (INT, QI, 8, 1)
RVV_FRACT_MODE (INT, HI, 2, 2)
RVV_FRACT_MODE (INT, HI, 4, 2)
RVV_FRACT_MODE (FLOAT, BF, 2, 2)
RVV_FRACT_MODE (FLOAT, BF, 4, 2)
RVV_FRACT_MODE (FLOAT, HF, 2, 2)
RVV_FRACT_MODE (FLOAT, HF, 4, 2)
RVV_FRACT_MODE (INT, SI, 2, 4)
RVV_FRACT_MODE (FLOAT, SF, 2, 4)

/* Tuple modes for segment loads/stores according to NF.

   Tuple modes format: RVV<LMUL>x<NF><BASEMODE>

   When LMUL is MF8/MF4/MF2/M1, NF can be 2 ~ 8.
   When LMUL is M2, NF can be 2 ~ 4.
   When LMUL is M4, NF can be 4.  */

#define RVV_NF8_MODES(NF)                                                      \
  VECTOR_MODE_WITH_PREFIX (RVVMF8x, INT, QI, NF, 1);                           \
  VECTOR_MODE_WITH_PREFIX (RVVMF4x, INT, QI, NF, 1);                           \
  VECTOR_MODE_WITH_PREFIX (RVVMF2x, INT, QI, NF, 1);                           \
  VECTOR_MODE_WITH_PREFIX (RVVM1x, INT, QI, NF, 1);                            \
  VECTOR_MODE_WITH_PREFIX (RVVMF4x, INT, HI, NF, 1);                           \
  VECTOR_MODE_WITH_PREFIX (RVVMF2x, INT, HI, NF, 1);                           \
  VECTOR_MODE_WITH_PREFIX (RVVM1x, INT, HI, NF, 1);                            \
  VECTOR_MODE_WITH_PREFIX (RVVMF4x, FLOAT, BF, NF, 1);				\
  VECTOR_MODE_WITH_PREFIX (RVVMF2x, FLOAT, BF, NF, 1);				\
  VECTOR_MODE_WITH_PREFIX (RVVM1x, FLOAT, BF, NF, 1);				\
  VECTOR_MODE_WITH_PREFIX (RVVMF4x, FLOAT, HF, NF, 1);                         \
  VECTOR_MODE_WITH_PREFIX (RVVMF2x, FLOAT, HF, NF, 1);                         \
  VECTOR_MODE_WITH_PREFIX (RVVM1x, FLOAT, HF, NF, 1);                          \
  VECTOR_MODE_WITH_PREFIX (RVVMF2x, INT, SI, NF, 1);                           \
  VECTOR_MODE_WITH_PREFIX (RVVM1x, INT, SI, NF, 1);                            \
  VECTOR_MODE_WITH_PREFIX (RVVMF2x, FLOAT, SF, NF, 1);                         \
  VECTOR_MODE_WITH_PREFIX (RVVM1x, FLOAT, SF, NF, 1);                          \
  VECTOR_MODE_WITH_PREFIX (RVVM1x, INT, DI, NF, 1);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM1x, FLOAT, DF, NF, 1);                          \
                                                                               \
  ADJUST_NUNITS (RVVMF8x##NF##QI,                                              \
		 riscv_v_adjust_nunits (RVVMF8x##NF##QImode, true, 8, NF));    \
  ADJUST_NUNITS (RVVMF4x##NF##QI,                                              \
		 riscv_v_adjust_nunits (RVVMF4x##NF##QImode, true, 4, NF));    \
  ADJUST_NUNITS (RVVMF2x##NF##QI,                                              \
		 riscv_v_adjust_nunits (RVVMF2x##NF##QImode, true, 2, NF));    \
  ADJUST_NUNITS (RVVM1x##NF##QI,                                               \
		 riscv_v_adjust_nunits (RVVM1x##NF##QImode, false, 1, NF));    \
  ADJUST_NUNITS (RVVMF4x##NF##HI,                                              \
		 riscv_v_adjust_nunits (RVVMF4x##NF##HImode, true, 4, NF));    \
  ADJUST_NUNITS (RVVMF2x##NF##HI,                                              \
		 riscv_v_adjust_nunits (RVVMF2x##NF##HImode, true, 2, NF));    \
  ADJUST_NUNITS (RVVM1x##NF##HI,                                               \
		 riscv_v_adjust_nunits (RVVM1x##NF##HImode, false, 1, NF));    \
  ADJUST_NUNITS (RVVMF4x##NF##BF,						\
		 riscv_v_adjust_nunits (RVVMF4x##NF##BFmode, true, 4, NF));    \
  ADJUST_NUNITS (RVVMF2x##NF##BF,						\
		 riscv_v_adjust_nunits (RVVMF2x##NF##BFmode, true, 2, NF));    \
  ADJUST_NUNITS (RVVM1x##NF##BF,						\
		 riscv_v_adjust_nunits (RVVM1x##NF##BFmode, false, 1, NF));    \
  ADJUST_NUNITS (RVVMF4x##NF##HF,                                              \
		 riscv_v_adjust_nunits (RVVMF4x##NF##HFmode, true, 4, NF));    \
  ADJUST_NUNITS (RVVMF2x##NF##HF,                                              \
		 riscv_v_adjust_nunits (RVVMF2x##NF##HFmode, true, 2, NF));    \
  ADJUST_NUNITS (RVVM1x##NF##HF,                                               \
		 riscv_v_adjust_nunits (RVVM1x##NF##HFmode, false, 1, NF));    \
  ADJUST_NUNITS (RVVMF2x##NF##SI,                                              \
		 riscv_v_adjust_nunits (RVVMF2x##NF##SImode, true, 2, NF));    \
  ADJUST_NUNITS (RVVM1x##NF##SI,                                               \
		 riscv_v_adjust_nunits (RVVM1x##NF##SImode, false, 1, NF));    \
  ADJUST_NUNITS (RVVMF2x##NF##SF,                                              \
		 riscv_v_adjust_nunits (RVVMF2x##NF##SFmode, true, 2, NF));    \
  ADJUST_NUNITS (RVVM1x##NF##SF,                                               \
		 riscv_v_adjust_nunits (RVVM1x##NF##SFmode, false, 1, NF));    \
  ADJUST_NUNITS (RVVM1x##NF##DI,                                               \
		 riscv_v_adjust_nunits (RVVM1x##NF##DImode, false, 1, NF));    \
  ADJUST_NUNITS (RVVM1x##NF##DF,                                               \
		 riscv_v_adjust_nunits (RVVM1x##NF##DFmode, false, 1, NF));    \
                                                                               \
  ADJUST_ALIGNMENT (RVVMF8x##NF##QI, 1);                                       \
  ADJUST_ALIGNMENT (RVVMF4x##NF##QI, 1);                                       \
  ADJUST_ALIGNMENT (RVVMF2x##NF##QI, 1);                                       \
  ADJUST_ALIGNMENT (RVVM1x##NF##QI, 1);                                        \
  ADJUST_ALIGNMENT (RVVMF4x##NF##HI, 2);                                       \
  ADJUST_ALIGNMENT (RVVMF2x##NF##HI, 2);                                       \
  ADJUST_ALIGNMENT (RVVM1x##NF##HI, 2);                                        \
  ADJUST_ALIGNMENT (RVVMF4x##NF##BF, 2);					\
  ADJUST_ALIGNMENT (RVVMF2x##NF##BF, 2);					\
  ADJUST_ALIGNMENT (RVVM1x##NF##BF, 2);					\
  ADJUST_ALIGNMENT (RVVMF4x##NF##HF, 2);                                       \
  ADJUST_ALIGNMENT (RVVMF2x##NF##HF, 2);                                       \
  ADJUST_ALIGNMENT (RVVM1x##NF##HF, 2);                                        \
  ADJUST_ALIGNMENT (RVVMF2x##NF##SI, 4);                                       \
  ADJUST_ALIGNMENT (RVVM1x##NF##SI, 4);                                        \
  ADJUST_ALIGNMENT (RVVMF2x##NF##SF, 4);                                       \
  ADJUST_ALIGNMENT (RVVM1x##NF##SF, 4);                                        \
  ADJUST_ALIGNMENT (RVVM1x##NF##DI, 8);                                        \
  ADJUST_ALIGNMENT (RVVM1x##NF##DF, 8);

RVV_NF8_MODES (8)
RVV_NF8_MODES (7)
RVV_NF8_MODES (6)
RVV_NF8_MODES (5)
RVV_NF8_MODES (4)
RVV_NF8_MODES (3)
RVV_NF8_MODES (2)

#define RVV_NF4_MODES(NF)                                                      \
  VECTOR_MODE_WITH_PREFIX (RVVM2x, INT, QI, NF, 1);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM2x, INT, HI, NF, 1);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM2x, FLOAT, BF, NF, 1);				\
  VECTOR_MODE_WITH_PREFIX (RVVM2x, FLOAT, HF, NF, 1);                          \
  VECTOR_MODE_WITH_PREFIX (RVVM2x, INT, SI, NF, 1);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM2x, FLOAT, SF, NF, 1);                          \
  VECTOR_MODE_WITH_PREFIX (RVVM2x, INT, DI, NF, 1);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM2x, FLOAT, DF, NF, 1);                          \
                                                                               \
  ADJUST_NUNITS (RVVM2x##NF##QI,                                               \
		 riscv_v_adjust_nunits (RVVM2x##NF##QImode, false, 2, NF));    \
  ADJUST_NUNITS (RVVM2x##NF##HI,                                               \
		 riscv_v_adjust_nunits (RVVM2x##NF##HImode, false, 2, NF));    \
  ADJUST_NUNITS (RVVM2x##NF##BF,						\
		 riscv_v_adjust_nunits (RVVM2x##NF##BFmode, false, 2, NF));    \
  ADJUST_NUNITS (RVVM2x##NF##HF,                                               \
		 riscv_v_adjust_nunits (RVVM2x##NF##HFmode, false, 2, NF));    \
  ADJUST_NUNITS (RVVM2x##NF##SI,                                               \
		 riscv_v_adjust_nunits (RVVM2x##NF##SImode, false, 2, NF));    \
  ADJUST_NUNITS (RVVM2x##NF##SF,                                               \
		 riscv_v_adjust_nunits (RVVM2x##NF##SFmode, false, 2, NF));    \
  ADJUST_NUNITS (RVVM2x##NF##DI,                                               \
		 riscv_v_adjust_nunits (RVVM2x##NF##DImode, false, 2, NF));    \
  ADJUST_NUNITS (RVVM2x##NF##DF,                                               \
		 riscv_v_adjust_nunits (RVVM2x##NF##DFmode, false, 2, NF));    \
                                                                               \
  ADJUST_ALIGNMENT (RVVM2x##NF##QI, 1);                                        \
  ADJUST_ALIGNMENT (RVVM2x##NF##HI, 2);                                        \
  ADJUST_ALIGNMENT (RVVM2x##NF##BF, 2);					\
  ADJUST_ALIGNMENT (RVVM2x##NF##HF, 2);                                        \
  ADJUST_ALIGNMENT (RVVM2x##NF##SI, 4);                                        \
  ADJUST_ALIGNMENT (RVVM2x##NF##SF, 4);                                        \
  ADJUST_ALIGNMENT (RVVM2x##NF##DI, 8);                                        \
  ADJUST_ALIGNMENT (RVVM2x##NF##DF, 8);

RVV_NF4_MODES (2)
RVV_NF4_MODES (3)
RVV_NF4_MODES (4)

#define RVV_NF2_MODES(NF)                                                      \
  VECTOR_MODE_WITH_PREFIX (RVVM4x, INT, QI, NF, 1);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM4x, INT, HI, NF, 1);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM4x, FLOAT, BF, NF, 1);				\
  VECTOR_MODE_WITH_PREFIX (RVVM4x, FLOAT, HF, NF, 1);                          \
  VECTOR_MODE_WITH_PREFIX (RVVM4x, INT, SI, NF, 1);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM4x, FLOAT, SF, NF, 1);                          \
  VECTOR_MODE_WITH_PREFIX (RVVM4x, INT, DI, NF, 1);                            \
  VECTOR_MODE_WITH_PREFIX (RVVM4x, FLOAT, DF, NF, 1);                          \
                                                                               \
  ADJUST_NUNITS (RVVM4x##NF##QI,                                               \
		 riscv_v_adjust_nunits (RVVM4x##NF##QImode, false, 4, NF));    \
  ADJUST_NUNITS (RVVM4x##NF##HI,                                               \
		 riscv_v_adjust_nunits (RVVM4x##NF##HImode, false, 4, NF));    \
  ADJUST_NUNITS (RVVM4x##NF##BF,						\
		 riscv_v_adjust_nunits (RVVM4x##NF##BFmode, false, 4, NF));    \
  ADJUST_NUNITS (RVVM4x##NF##HF,                                               \
		 riscv_v_adjust_nunits (RVVM4x##NF##HFmode, false, 4, NF));    \
  ADJUST_NUNITS (RVVM4x##NF##SI,                                               \
		 riscv_v_adjust_nunits (RVVM4x##NF##SImode, false, 4, NF));    \
  ADJUST_NUNITS (RVVM4x##NF##SF,                                               \
		 riscv_v_adjust_nunits (RVVM4x##NF##SFmode, false, 4, NF));    \
  ADJUST_NUNITS (RVVM4x##NF##DI,                                               \
		 riscv_v_adjust_nunits (RVVM4x##NF##DImode, false, 4, NF));    \
  ADJUST_NUNITS (RVVM4x##NF##DF,                                               \
		 riscv_v_adjust_nunits (RVVM4x##NF##DFmode, false, 4, NF));    \
                                                                               \
  ADJUST_ALIGNMENT (RVVM4x##NF##QI, 1);                                        \
  ADJUST_ALIGNMENT (RVVM4x##NF##HI, 2);                                        \
  ADJUST_ALIGNMENT (RVVM4x##NF##BF, 2);					\
  ADJUST_ALIGNMENT (RVVM4x##NF##HF, 2);                                        \
  ADJUST_ALIGNMENT (RVVM4x##NF##SI, 4);                                        \
  ADJUST_ALIGNMENT (RVVM4x##NF##SF, 4);                                        \
  ADJUST_ALIGNMENT (RVVM4x##NF##DI, 8);                                        \
  ADJUST_ALIGNMENT (RVVM4x##NF##DF, 8);

RVV_NF2_MODES (2)

/* VLS modes used as SIMD auto-vectorization for epilogue. We know the
   return type of GET_MODE_BITSIZE is poly_uint16 or unsigned short.
   The maximum bitsize of all vector modes is 65536 = (8192 (LMUL1) * 8),
   even though RISC-V 'V' ISA spec allow maximum bitsize = 65536 * 8.

   Note: We don't enable of vector modes of TI/TF.  */

VECTOR_BOOL_MODE (V1BI, 1, BI, 1);	   /*    V1BI */
VECTOR_BOOL_MODE (V2BI, 2, BI, 1);	   /*    V2BI */
VECTOR_BOOL_MODE (V4BI, 4, BI, 1);	   /*    V4BI */
VECTOR_BOOL_MODE (V8BI, 8, BI, 1);	   /*    V8BI */
VECTOR_BOOL_MODE (V16BI, 16, BI, 2);	   /*   V16BI */
VECTOR_BOOL_MODE (V32BI, 32, BI, 4);	   /*   V32BI */
VECTOR_BOOL_MODE (V64BI, 64, BI, 8);	   /*   V64BI */
VECTOR_BOOL_MODE (V128BI, 128, BI, 16);	   /*  V128BI */
VECTOR_BOOL_MODE (V256BI, 256, BI, 32);	   /*  V256BI */
VECTOR_BOOL_MODE (V512BI, 512, BI, 64);	   /*  V512BI */
VECTOR_BOOL_MODE (V1024BI, 1024, BI, 128); /* V1024BI */
VECTOR_BOOL_MODE (V2048BI, 2048, BI, 256); /* V2048BI */
VECTOR_BOOL_MODE (V4096BI, 4096, BI, 512); /* V4096BI */

ADJUST_ALIGNMENT (V1BI, 1);
ADJUST_ALIGNMENT (V2BI, 1);
ADJUST_ALIGNMENT (V4BI, 1);
ADJUST_ALIGNMENT (V8BI, 1);
ADJUST_ALIGNMENT (V16BI, 1);
ADJUST_ALIGNMENT (V32BI, 1);
ADJUST_ALIGNMENT (V64BI, 1);
ADJUST_ALIGNMENT (V128BI, 1);
ADJUST_ALIGNMENT (V256BI, 1);
ADJUST_ALIGNMENT (V512BI, 1);
ADJUST_ALIGNMENT (V1024BI, 1);
ADJUST_ALIGNMENT (V2048BI, 1);
ADJUST_ALIGNMENT (V4096BI, 1);

ADJUST_PRECISION (V1BI, 1);
ADJUST_PRECISION (V2BI, 2);
ADJUST_PRECISION (V4BI, 4);
ADJUST_PRECISION (V8BI, 8);
ADJUST_PRECISION (V16BI, 16);
ADJUST_PRECISION (V32BI, 32);
ADJUST_PRECISION (V64BI, 64);
ADJUST_PRECISION (V128BI, 128);
ADJUST_PRECISION (V256BI, 256);
ADJUST_PRECISION (V512BI, 512);
ADJUST_PRECISION (V1024BI, 1024);
ADJUST_PRECISION (V2048BI, 2048);
ADJUST_PRECISION (V4096BI, 4096);

#define VLS_MODES(NBYTES)                                                      \
  VECTOR_MODE_WITH_PREFIX (V, INT, QI, NBYTES, 1);                             \
  VECTOR_MODE_WITH_PREFIX (V, INT, HI, NBYTES / 2, 1);                         \
  VECTOR_MODE_WITH_PREFIX (V, INT, SI, NBYTES / 4, 1);                         \
  VECTOR_MODE_WITH_PREFIX (V, INT, DI, NBYTES / 8, 1);                         \
  VECTOR_MODE_WITH_PREFIX (V, FLOAT, HF, NBYTES / 2, 1);                       \
  VECTOR_MODE_WITH_PREFIX (V, FLOAT, SF, NBYTES / 4, 1);                       \
  VECTOR_MODE_WITH_PREFIX (V, FLOAT, DF, NBYTES / 8, 1);

VECTOR_MODE_WITH_PREFIX (V, INT, QI, 1, 1);   /* V1QI */
VECTOR_MODE_WITH_PREFIX (V, INT, HI, 1, 1);   /* V1HI */
VECTOR_MODE_WITH_PREFIX (V, INT, SI, 1, 1);   /* V1SI */
VECTOR_MODE_WITH_PREFIX (V, INT, DI, 1, 1);   /* V1DI */
VECTOR_MODE_WITH_PREFIX (V, FLOAT, HF, 1, 1); /* V1HF */
VECTOR_MODE_WITH_PREFIX (V, FLOAT, SF, 1, 1); /* V1SF */
VECTOR_MODE_WITH_PREFIX (V, FLOAT, DF, 1, 1); /* V1DF */
VECTOR_MODE_WITH_PREFIX (V, INT, QI, 2, 1);   /* V2QI */
VECTOR_MODE_WITH_PREFIX (V, INT, QI, 4, 1);   /* V4QI */
VECTOR_MODE_WITH_PREFIX (V, INT, QI, 8, 1);   /* V8QI */
VECTOR_MODE_WITH_PREFIX (V, INT, HI, 2, 1);   /* V2HI */
VECTOR_MODE_WITH_PREFIX (V, INT, HI, 4, 1);   /* V4HI */
VECTOR_MODE_WITH_PREFIX (V, FLOAT, HF, 2, 1); /* V2HF */
VECTOR_MODE_WITH_PREFIX (V, FLOAT, HF, 4, 1); /* V4HF */
VECTOR_MODE_WITH_PREFIX (V, INT, SI, 2, 1);   /* V2SI */
VECTOR_MODE_WITH_PREFIX (V, FLOAT, SF, 2, 1); /* V2SF */
VLS_MODES (16);	  /*   V16QI    V8HI    V4SI   V2DI    V8HF    V4SF   V2DF */
VLS_MODES (32);	  /*   V32QI   V16HI    V8SI   V4DI   V16HF    V8SF   V4DF */
VLS_MODES (64);	  /*   V64QI   V32HI   V16SI   V8DI   V32HF   V16SF   V8DF */
VLS_MODES (128);  /*  V128QI   V64HI   V32SI  V16DI   V64HF   V32SF  V16DF */
VLS_MODES (256);  /*  V256QI  V128HI   V64SI  V32DI  V128HF   V64SF  V32DF */
VLS_MODES (512);  /*  V512QI  V256HI  V128SI  V64DI  V256HF  V128SF  V64DF */
VLS_MODES (1024); /* V1024QI  V512HI  V256SI V128DI  V512HF  V256SF V128DF */
VLS_MODES (2048); /* V2048QI V1024HI  V512SI V256DI V1024HF  V512SF V256DF */
VLS_MODES (4096); /* V4096QI V2048HI V1024SI V512DI V2048HF V1024SF V512DF */

/* TODO: According to RISC-V 'V' ISA spec, the maximum vector length can
   be 65536 for a single vector register which means the vector mode in
   GCC can be maximum = 65536 * 8 bits (LMUL=8).
   However, 'GET_MODE_SIZE' is using poly_uint16/unsigned short which will
   overflow if we specify vector-length = 65536. To support this feature,
   we need to change the codes outside the RISC-V port. We will support it in
   the future.  */
#define MAX_BITSIZE_MODE_ANY_MODE (4096 * 8)

/* Coefficient 1 is multiplied by the number of 64-bit/32-bit chunks in a vector
   minus one.  */
#define NUM_POLY_INT_COEFFS 2
