aboutsummaryrefslogtreecommitdiff
path: root/gcc/testsuite/gcc.target
diff options
context:
space:
mode:
Diffstat (limited to 'gcc/testsuite/gcc.target')
-rw-r--r--gcc/testsuite/gcc.target/aarch64/cmpbr.c40
-rw-r--r--gcc/testsuite/gcc.target/aarch64/pr121300.c9
-rw-r--r--gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c12
-rw-r--r--gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c8
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/mask_load_2.c23
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_1.c44
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_2.c20
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_1.c44
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_2.c20
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_1.c51
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_2.c24
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_1.c51
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_2.c24
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_cvtf_1.c51
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fabs_1.c37
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_1.c62
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_2.c28
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvt_1.c41
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvtz_1.c55
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_1.c47
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_2.c22
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_1.c53
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_2.c24
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_1.c53
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_2.c24
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_1.c51
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_2.c22
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_1.c51
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_2.c22
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_1.c50
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_2.c22
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fneg_1.c39
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c51
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_2.c22
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c51
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_2.c22
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_1.c37
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_2.c18
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinti_1.c37
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintm_1.c37
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintp_1.c37
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintx_1.c37
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintz_1.c37
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_1.c56
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_2.c26
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fabs_1.c28
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_1.c52
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c26
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_1.c38
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_2.c15
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_3.c13
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_1.c45
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_2.c20
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_1.c46
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_2.c20
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_1.c38
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_2.c15
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_1.c38
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_2.c15
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_1.c43
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c18
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fneg_1.c30
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_1.c38
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_2.c15
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_1.c38
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_2.c15
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_1.c31
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_2.c15
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_1.c31
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_2.c15
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_1.c31
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_2.c15
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_1.c31
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_2.c15
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_1.c31
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_2.c15
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_1.c31
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_2.c15
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_1.c46
-rw-r--r--gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c20
-rw-r--r--gcc/testsuite/gcc.target/i386/pr119795.c26
-rw-r--r--gcc/testsuite/gcc.target/i386/pr120427-5.c10
-rw-r--r--gcc/testsuite/gcc.target/i386/pr121208-1a.c15
-rw-r--r--gcc/testsuite/gcc.target/i386/pr121208-1b.c4
-rw-r--r--gcc/testsuite/gcc.target/i386/pr121208-2a.c17
-rw-r--r--gcc/testsuite/gcc.target/i386/pr121208-2b.c4
-rw-r--r--gcc/testsuite/gcc.target/i386/pr121208-3a.c17
-rw-r--r--gcc/testsuite/gcc.target/i386/pr121208-3b.c4
-rw-r--r--gcc/testsuite/gcc.target/i386/pr121274.c24
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_100.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_100a.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_100f.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_101.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_101a.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_101f.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_103.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_103a.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_103f.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_120.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_120a.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_120f.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_121.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_121a.c19
-rw-r--r--gcc/testsuite/gcc.target/nvptx/march-map=sm_121f.c19
-rw-r--r--gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c10
-rw-r--r--gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c2
-rw-r--r--gcc/testsuite/gcc.target/riscv/interrupt-rnmi.c11
-rw-r--r--gcc/testsuite/gcc.target/riscv/interrupt-umode.c8
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c2
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c5
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c2
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c7
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c2
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c2
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c2
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c5
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i16.c23
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i32.c23
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i64.c23
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i8.c23
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u16.c23
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u32.c23
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u64.c23
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u8.c23
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm.h28
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h62
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h392
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i16.c17
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i32.c17
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i64.c17
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i8.c17
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u16.c17
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u32.c17
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u64.c17
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u8.c17
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c4
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c6
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c5
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c1
-rw-r--r--gcc/testsuite/gcc.target/riscv/rvv/pr121073.c12
-rw-r--r--gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u32.c11
-rw-r--r--gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u16.c11
-rw-r--r--gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u32.c11
-rw-r--r--gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u16-from-u64.c11
-rw-r--r--gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u32-from-u64.c11
-rw-r--r--gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u8-from-u64.c11
-rw-r--r--gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u32.c16
-rw-r--r--gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c2
-rw-r--r--gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c2
-rw-r--r--gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u16.c16
-rw-r--r--gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u32.c16
-rw-r--r--gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c2
-rw-r--r--gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c6
-rw-r--r--gcc/testsuite/gcc.target/s390/spaceship-fp-1.c23
-rw-r--r--gcc/testsuite/gcc.target/s390/spaceship-fp-2.c23
-rw-r--r--gcc/testsuite/gcc.target/s390/spaceship-fp-3.c23
-rw-r--r--gcc/testsuite/gcc.target/s390/spaceship-fp-4.c53
-rw-r--r--gcc/testsuite/gcc.target/s390/spaceship-int-1.c30
-rw-r--r--gcc/testsuite/gcc.target/s390/spaceship-int-2.c24
-rw-r--r--gcc/testsuite/gcc.target/s390/spaceship-int-3.c21
204 files changed, 4218 insertions, 98 deletions
diff --git a/gcc/testsuite/gcc.target/aarch64/cmpbr.c b/gcc/testsuite/gcc.target/aarch64/cmpbr.c
index a86af9d..34630f9 100644
--- a/gcc/testsuite/gcc.target/aarch64/cmpbr.c
+++ b/gcc/testsuite/gcc.target/aarch64/cmpbr.c
@@ -121,7 +121,7 @@ FAR_BRANCH(u64, 42);
/*
** u8_x0_eq_x1:
-** cbbeq w1, w0, .L([0-9]+)
+** cbbeq (?:w1, w0|w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -129,7 +129,7 @@ FAR_BRANCH(u64, 42);
/*
** u8_x0_ne_x1:
-** cbbne w1, w0, .L([0-9]+)
+** cbbne (?:w1, w0|w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -137,7 +137,7 @@ FAR_BRANCH(u64, 42);
/*
** u8_x0_ult_x1:
-** cbbhi w1, w0, .L([0-9]+)
+** (?:cbbhi w1, w0|cbblo w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -145,7 +145,7 @@ FAR_BRANCH(u64, 42);
/*
** u8_x0_ule_x1:
-** cbbhs w1, w0, .L([0-9]+)
+** (?:cbbhs w1, w0|cbbls w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -153,7 +153,7 @@ FAR_BRANCH(u64, 42);
/*
** u8_x0_ugt_x1:
-** cbblo w1, w0, .L([0-9]+)
+** (?:cbblo w1, w0|cbbhi w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -161,7 +161,7 @@ FAR_BRANCH(u64, 42);
/*
** u8_x0_uge_x1:
-** cbbls w1, w0, .L([0-9]+)
+** (?:cbbls w1, w0|cbbhs w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -169,7 +169,7 @@ FAR_BRANCH(u64, 42);
/*
** i8_x0_slt_x1:
-** cbbgt w1, w0, .L([0-9]+)
+** (?:cbbgt w1, w0|cbblt w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -177,7 +177,7 @@ FAR_BRANCH(u64, 42);
/*
** i8_x0_sle_x1:
-** cbbge w1, w0, .L([0-9]+)
+** (?:cbbge w1, w0|cbble w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -185,7 +185,7 @@ FAR_BRANCH(u64, 42);
/*
** i8_x0_sgt_x1:
-** cbblt w1, w0, .L([0-9]+)
+** (?:cbblt w1, w0|cbbgt w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -193,7 +193,7 @@ FAR_BRANCH(u64, 42);
/*
** i8_x0_sge_x1:
-** cbble w1, w0, .L([0-9]+)
+** (?:cbble w1, w0|cbbge w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -201,7 +201,7 @@ FAR_BRANCH(u64, 42);
/*
** u16_x0_eq_x1:
-** cbheq w1, w0, .L([0-9]+)
+** cbheq (?:w1, w0|w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -209,7 +209,7 @@ FAR_BRANCH(u64, 42);
/*
** u16_x0_ne_x1:
-** cbhne w0|w1, w1|w0, .L([0-9]+)
+** cbhne (?:w1, w0|w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -217,7 +217,7 @@ FAR_BRANCH(u64, 42);
/*
** u16_x0_ult_x1:
-** cbhhi w1, w0, .L([0-9]+)
+** (?:cbhhi w1, w0|cbhlo w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -225,7 +225,7 @@ FAR_BRANCH(u64, 42);
/*
** u16_x0_ule_x1:
-** cbhhs w1, w0, .L([0-9]+)
+** (?:cbhhs w1, w0|cbhls w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -233,7 +233,7 @@ FAR_BRANCH(u64, 42);
/*
** u16_x0_ugt_x1:
-** cbhlo w1, w0, .L([0-9]+)
+** (?:cbhlo w1, w0|cbhhi w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -241,7 +241,7 @@ FAR_BRANCH(u64, 42);
/*
** u16_x0_uge_x1:
-** cbhls w1, w0, .L([0-9]+)
+** (?:cbhls w1, w0|cbhhs w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -249,7 +249,7 @@ FAR_BRANCH(u64, 42);
/*
** i16_x0_slt_x1:
-** cbhgt w1, w0, .L([0-9]+)
+** (?:cbhgt w1, w0|cbhlt w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -257,7 +257,7 @@ FAR_BRANCH(u64, 42);
/*
** i16_x0_sle_x1:
-** cbhge w1, w0, .L([0-9]+)
+** (?:cbhge w1, w0|cbhle w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -265,7 +265,7 @@ FAR_BRANCH(u64, 42);
/*
** i16_x0_sgt_x1:
-** cbhlt w1, w0, .L([0-9]+)
+** (?:cbhlt w1, w0|cbhgt w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
@@ -273,7 +273,7 @@ FAR_BRANCH(u64, 42);
/*
** i16_x0_sge_x1:
-** cbhle w1, w0, .L([0-9]+)
+** (?:cbhle w1, w0|cbhge w0, w1), .L([0-9]+)
** b not_taken
** .L\1:
** b taken
diff --git a/gcc/testsuite/gcc.target/aarch64/pr121300.c b/gcc/testsuite/gcc.target/aarch64/pr121300.c
new file mode 100644
index 0000000..5f2cd9a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr121300.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-S -O3 -march=armv9-a+sme2" } */
+
+#include <arm_sme.h>
+
+svfloat16x2_t test (svfloat16x2_t zd, svfloat16x2_t zm) __arm_streaming
+{
+ return svamin_f16_x2 (zd, zm); // { dg-error "ACLE function .svamin_f16_x2. requires ISA extension .faminmax." }
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
index acd2e11..8fc1569 100644
--- a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
@@ -4,24 +4,24 @@
/*
** uadd:
-** dup v([0-9]+).8b, w1
-** dup v([0-9]+).8b, w0
+** dup v([0-9]+).8b, w[01]
+** dup v([0-9]+).8b, w[01]
** uqadd b([0-9]+), (?:b\2, b\1|b\1, b\2)
** umov w0, v\3.b\[0\]
** ret
*/
/*
** uadd2:
-** dup v([0-9]+).8b, w1
-** dup v([0-9]+).8b, w0
+** dup v([0-9]+).8b, w[01]
+** dup v([0-9]+).8b, w[01]
** uqadd b([0-9]+), (?:b\2, b\1|b\1, b\2)
** umov w0, v\3.b\[0\]
** ret
*/
/*
** usub: { xfail *-*-* }
-** dup v([0-9]+).8b, w1
-** dup v([0-9]+).8b, w0
+** dup v([0-9]+).8b, w[01]
+** dup v([0-9]+).8b, w[01]
** uqsub b([0-9]+), b\1, b\2
** umov w0, v\3.b\[0\]
** ret
diff --git a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
index 86c88f8..dd0fefa 100644
--- a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
@@ -4,16 +4,16 @@
/*
** uadd:
-** dup v([0-9]+).4h, w1
-** dup v([0-9]+).4h, w0
+** dup v([0-9]+).4h, w[01]
+** dup v([0-9]+).4h, w[01]
** uqadd h([0-9]+), (?:h\2, h\1|h\1, h\2)
** umov w0, v\3.h\[0\]
** ret
*/
/*
** uadd2:
-** dup v([0-9]+).4h, w1
-** dup v([0-9]+).4h, w0
+** dup v([0-9]+).4h, w[01]
+** dup v([0-9]+).4h, w[01]
** uqadd h([0-9]+), (?:h\2, h\1|h\1, h\2)
** umov w0, v\3.h\[0\]
** ret
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mask_load_2.c b/gcc/testsuite/gcc.target/aarch64/sve/mask_load_2.c
new file mode 100644
index 0000000..66d9510
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/mask_load_2.c
@@ -0,0 +1,23 @@
+// { dg-do compile }
+// { dg-options "-march=armv8-a+sve -msve-vector-bits=128 -O3" }
+
+typedef struct Array {
+ int elems[3];
+} Array;
+
+int loop(Array **pp, int len, int idx) {
+ int nRet = 0;
+
+ #pragma GCC unroll 0
+ for (int i = 0; i < len; i++) {
+ Array *p = pp[i];
+ if (p) {
+ nRet += p->elems[idx];
+ }
+ }
+
+ return nRet;
+}
+
+// { dg-final { scan-assembler-times {ld1w\tz[0-9]+\.d, p[0-7]/z} 1 } }
+// { dg-final { scan-assembler-times {add\tz[0-9]+\.s, p[0-7]/m} 1 } }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_1.c
new file mode 100644
index 0000000..e6aa047
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_1.c
@@ -0,0 +1,44 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i], RHS) > c[i]) \
+ out[i] = 3; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (__builtin_fmaxf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fmaxf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fmaxf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_2.c
new file mode 100644
index 0000000..87125a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_builtin_fmax_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_1.c
new file mode 100644
index 0000000..b9fded0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_1.c
@@ -0,0 +1,44 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i], RHS) > c[i]) \
+ out[i] = 3; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (__builtin_fminf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fminf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fminf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_2.c
new file mode 100644
index 0000000..5923b67
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_builtin_fmin_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_1.c
new file mode 100644
index 0000000..d328b37
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE) \
+ void \
+ f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE1 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)
+
+TEST_ALL (__builtin_fmaxf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fmaxf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fmaxf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_2.c
new file mode 100644
index 0000000..f84ded5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_builtin_fmax_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_1.c
new file mode 100644
index 0000000..1821f03
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE) \
+ void \
+ f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE1 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)
+
+TEST_ALL (__builtin_fminf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fminf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fminf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_2.c
new file mode 100644
index 0000000..bceddf9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_builtin_fmin_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_cvtf_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_cvtf_1.c
new file mode 100644
index 0000000..fa4dd15
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_cvtf_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define COND_CVT(TYPE0, TYPE1, TYPE2, COUNT) \
+ void \
+ test_##TYPE0##_##TYPE1##_##TYPE2 (TYPE0 *__restrict out, \
+ TYPE1 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE2 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? (TYPE0)a[i] : b[i]; \
+ }
+
+#define TEST_CVTF(PFX, T) \
+ T (_Float16, PFX##int16_t, uint64_t, 32) \
+ T (_Float16, PFX##int16_t, uint32_t, 64) \
+ T (_Float16, PFX##int32_t, uint64_t, 32) \
+ T (_Float16, PFX##int32_t, uint32_t, 64) \
+ T (_Float16, PFX##int64_t, uint64_t, 32) \
+ T (float, PFX##int32_t, uint64_t, 32) \
+ T (float, PFX##int64_t, uint64_t, 32)
+
+#define TEST_ALL(T) \
+ TEST_CVTF (, T) \
+ TEST_CVTF (u, T)
+
+TEST_ALL (COND_CVT)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 6 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 8 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fabs_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fabs_1.c
new file mode 100644
index 0000000..d959aa9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fabs_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p, \
+ TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i]) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_fabsf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fabsf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fabsf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_1.c
new file mode 100644
index 0000000..666cf89
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_1.c
@@ -0,0 +1,62 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+#define imm_p5 0.5
+
+#define ADD(A, B) A + B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, NAME, RHS, MERGE) \
+ void \
+ f_##TYPE0##_##TYPE1##_##NAME##_##MERGE (TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE1 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b[i], a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b[i], b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b[i], c_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, one, 1, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, one, 1, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, none, -1, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, none, -1, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, p5, 0.5, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, p5, 0.5, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, np5, -0.5, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, np5, -0.5, b_i)
+
+TEST_ALL (ADD, _Float16, uint64_t, 32)
+
+TEST_ALL (ADD, _Float16, uint32_t, 64)
+
+TEST_ALL (ADD, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 19 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 19 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 19 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 5 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 10 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_2.c
new file mode 100644
index 0000000..e59864b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_2.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fadd_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 11 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 22 } } */
+/* { dg-final { scan-assembler-times {\tand} 33 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 19 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 19 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 19 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 5 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 10 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvt_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvt_1.c
new file mode 100644
index 0000000..3caae19
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvt_1.c
@@ -0,0 +1,41 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define COND_CVT(TYPE0, TYPE1, TYPE2, COUNT) \
+ void \
+ test_##TYPE0##_##TYPE1##_##TYPE2 (TYPE0 *__restrict out, \
+ TYPE1 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE2 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? (TYPE0)a[i] : b[i]; \
+ }
+
+#define TEST_FCVT(T) \
+ T (_Float16, float, uint64_t, 32) \
+ T (_Float16, float, uint32_t, 64) \
+ T (_Float16, double, uint64_t, 32) \
+ T (float, double, uint64_t, 32) \
+ T (float, _Float16, uint64_t, 32) \
+ T (float, _Float16, uint32_t, 64) \
+ T (double, _Float16, uint64_t,32) \
+ T (double, float, uint64_t, 32)
+
+TEST_FCVT (COND_CVT)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvtz_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvtz_1.c
new file mode 100644
index 0000000..426d3af
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvtz_1.c
@@ -0,0 +1,55 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define COND_CVT(TYPE0, TYPE1, TYPE2, COUNT) \
+ void \
+ test_##TYPE0##_##TYPE1##_##TYPE2 (TYPE0 *__restrict out, \
+ TYPE1 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE2 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? (TYPE0)a[i] : b[i]; \
+ }
+
+#define TEST_FCVTZ(PFX, T) \
+ T (PFX##int16_t, _Float16, uint64_t, 32) \
+ T (PFX##int16_t, _Float16, uint32_t, 64) \
+ T (PFX##int32_t, _Float16, uint64_t, 32) \
+ T (PFX##int32_t, _Float16, uint32_t, 64) \
+ T (PFX##int64_t, _Float16, uint64_t, 32) \
+ T (PFX##int32_t, float, uint64_t, 32) \
+ T (PFX##int64_t, float, uint64_t, 32) \
+ T (PFX##int32_t, double, uint64_t, 32)
+
+#define TEST_ALL(T) \
+ TEST_FCVTZ (, T) \
+ TEST_FCVTZ (u, T)
+
+TEST_ALL (COND_CVT)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 6 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 8 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_1.c
new file mode 100644
index 0000000..ec5653e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_1.c
@@ -0,0 +1,47 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define DIV(A, B) A / B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE) \
+ void \
+ f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE1 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i)
+
+TEST_ALL (DIV, _Float16, uint64_t, 32)
+
+TEST_ALL (DIV, _Float16, uint32_t, 64)
+
+TEST_ALL (DIV, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_2.c
new file mode 100644
index 0000000..1ca3dbf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fdiv_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 6 } } */
+/* { dg-final { scan-assembler-times {\tand} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_1.c
new file mode 100644
index 0000000..d34872f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_1.c
@@ -0,0 +1,53 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-signed-zeros -ffinite-math-only -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define MAX(A, B) (A > B) ? A : B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE) \
+ void \
+ f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE1 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)
+
+TEST_ALL (MAX, _Float16, uint64_t, 32)
+
+TEST_ALL (MAX, _Float16, uint32_t, 64)
+
+TEST_ALL (MAX, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_2.c
new file mode 100644
index 0000000..282f3ed
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-signed-zeros -ffinite-math-only" } */
+
+#include "unpacked_cond_fmaxnm_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_1.c
new file mode 100644
index 0000000..d6c3c38
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_1.c
@@ -0,0 +1,53 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-signed-zeros -ffinite-math-only -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define MIN(A, B) (A < B) ? A : B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE) \
+ void \
+ f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE1 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)
+
+TEST_ALL (MIN, _Float16, uint64_t, 32)
+
+TEST_ALL (MIN, _Float16, uint32_t, 64)
+
+TEST_ALL (MIN, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_2.c
new file mode 100644
index 0000000..8226a6f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-signed-zeros -ffinite-math-only" } */
+
+#include "unpacked_cond_fminnm_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_1.c
new file mode 100644
index 0000000..cae9242
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF) __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF) __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE) \
+ void \
+ f_##TYPE0##_##TYPE1##_##MERGE (TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE1 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, c_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)
+
+TEST_ALL (FMLA (f16), _Float16, uint64_t, 32)
+
+TEST_ALL (FMLA (f16), _Float16, uint32_t, 64)
+
+TEST_ALL (FMLA (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_2.c
new file mode 100644
index 0000000..72e04a4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fmla_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tand} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_1.c
new file mode 100644
index 0000000..db0f818
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF) __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF) __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE) \
+ void \
+ f_##TYPE0##_##TYPE1##_##MERGE (TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE1 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, c_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)
+
+TEST_ALL (FMLS (f16), _Float16, uint64_t, 32)
+
+TEST_ALL (FMLS (f16), _Float16, uint32_t, 64)
+
+TEST_ALL (FMLS (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_2.c
new file mode 100644
index 0000000..3012052
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fmls_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tand} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_1.c
new file mode 100644
index 0000000..1ae7678
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_1.c
@@ -0,0 +1,50 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+#define imm_p5 0.5
+
+#define MUL(A, B) A * B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE) \
+ void \
+ f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE1 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, imm_p5, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, imm_p5, b_i)
+
+TEST_ALL (MUL, _Float16, uint64_t, 32)
+
+TEST_ALL (MUL, _Float16, uint32_t, 64)
+
+TEST_ALL (MUL, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 10 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_2.c
new file mode 100644
index 0000000..21713f5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fmul_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 10 } } */
+/* { dg-final { scan-assembler-times {\tand} 15 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 10 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fneg_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fneg_1.c
new file mode 100644
index 0000000..7280f4e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fneg_1.c
@@ -0,0 +1,39 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define NEG(X) -X
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p, \
+ TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i]) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (NEG, _Float16, uint64_t, 32)
+
+TEST_ALL (NEG, _Float16, uint32_t, 64)
+
+TEST_ALL (NEG, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c
new file mode 100644
index 0000000..07bab63
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF) __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF) __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE) \
+ void \
+ f_##TYPE0##_##TYPE1##_##MERGE (TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE1 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, c_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)
+
+TEST_ALL (FNMLA (f16), _Float16, uint64_t, 32)
+
+TEST_ALL (FNMLA (f16), _Float16, uint32_t, 64)
+
+TEST_ALL (FNMLA (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_2.c
new file mode 100644
index 0000000..daef4e49
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fnmla_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tand} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c
new file mode 100644
index 0000000..5526378
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF) __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF) __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE) \
+ void \
+ f_##TYPE0##_##TYPE1##_##MERGE (TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE1 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, c_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)
+
+TEST_ALL (FNMLS (f16), _Float16, uint64_t, 32)
+
+TEST_ALL (FNMLS (f16), _Float16, uint32_t, 64)
+
+TEST_ALL (FNMLS (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_2.c
new file mode 100644
index 0000000..8a8f348
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fnmls_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tand} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_1.c
new file mode 100644
index 0000000..ed4efb6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p, \
+ TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i]) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_roundf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_roundf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_roundf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_2.c
new file mode 100644
index 0000000..f20e2e6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_2.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -mtune=generic -ftree-vectorize" } */
+
+#include "unpacked_cond_frinta_1.c"
+
+/* Test that we don't drop SELs without -fno-trapping-math. */
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tsel\t} 6 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinti_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinti_1.c
new file mode 100644
index 0000000..d682d15
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinti_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p, \
+ TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i]) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_nearbyintf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_nearbyintf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_nearbyintf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintm_1.c
new file mode 100644
index 0000000..7d429b3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintm_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p, \
+ TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i]) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_floorf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_floorf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_floorf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintp_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintp_1.c
new file mode 100644
index 0000000..c6d0c8c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintp_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p, \
+ TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i]) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_ceilf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_ceilf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_ceilf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintx_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintx_1.c
new file mode 100644
index 0000000..b8afef1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintx_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p, \
+ TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i]) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_rintf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_rintf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_rintf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintz_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintz_1.c
new file mode 100644
index 0000000..d55279b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintz_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p, \
+ TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i]) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_truncf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_truncf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_truncf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_1.c
new file mode 100644
index 0000000..eafd169
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_1.c
@@ -0,0 +1,56 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+#define imm_p5 0.5
+
+#define SUBR(A, B) B - A
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE) \
+ void \
+ f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE1 *__restrict p) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, imm_p5, a_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, imm_p5, b_i)
+
+TEST_ALL (SUBR, _Float16, uint64_t, 32)
+
+TEST_ALL (SUBR, _Float16, uint32_t, 64)
+
+TEST_ALL (SUBR, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_2.c
new file mode 100644
index 0000000..cd7a0e1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_2.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fsubr_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fabs_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fabs_1.c
new file mode 100644
index 0000000..f09cfe8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fabs_1.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i]) > b[i]) \
+ out[i] = 3; \
+ }
+
+TEST_FN (__builtin_fabsf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_fabsf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_fabsf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_1.c
new file mode 100644
index 0000000..9675f56
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_1.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define ADD(A, B) A + B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, NAME, RHS) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##NAME (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i], (TYPE0)RHS) > c[i]) \
+ out[i] = 3; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b[i]) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, p5, 0.5) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, np5, -0.5) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, one, 1) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, none, -1)
+
+TEST_ALL (ADD, _Float16, uint64_t, 32)
+
+TEST_ALL (ADD, _Float16, uint32_t, 64)
+
+TEST_ALL (ADD, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 10 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 11 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 11 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 11 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
new file mode 100644
index 0000000..7a74efd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
@@ -0,0 +1,26 @@
+/* { dg-do compile }*/
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fadd_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 11 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 11 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 11 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_1.c
new file mode 100644
index 0000000..78d0d9c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define DIV(A, B) A / B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i], (TYPE0)RHS) > c[i]) \
+ out[i] = 3; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (DIV, _Float16, uint64_t, 32)
+
+TEST_ALL (DIV, _Float16, uint32_t, 64)
+
+TEST_ALL (DIV, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfdivr?\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr?\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_2.c
new file mode 100644
index 0000000..a8f70e1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fdiv_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfdivr?\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr?\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_3.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_3.c
new file mode 100644
index 0000000..ecd088f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_3.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-Ofast -moverride=sve_width=2048 -mlow-precision-div" } */
+
+#include "unpacked_fdiv_1.c"
+
+/* { dg-final { scan-assembler-not {\tfrecpe\tz[0-9]+\.h} } } */
+/* { dg-final { scan-assembler-not {\tfrecps\tz[0-9]+\.h} } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrecpe\tz[0-9]+\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrecps\tz[0-9]+\.s} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_1.c
new file mode 100644
index 0000000..5239e4b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_1.c
@@ -0,0 +1,45 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define MAX(A, B) (A > B) ? A : B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (c[i] = FN (a[i], RHS)) \
+ out[i] = 3; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (MAX, _Float16, uint64_t, 32)
+
+TEST_ALL (MAX, _Float16, uint32_t, 64)
+
+TEST_ALL (MAX, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_2.c
new file mode 100644
index 0000000..11aa7c0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -fno-trapping-math -moverride=sve_width=2048" } */
+
+#include "unpacked_fmaxnm_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_1.c
new file mode 100644
index 0000000..02a5f46
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_1.c
@@ -0,0 +1,46 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define MIN(A, B) (A < B) ? A : B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (c[i] = FN (a[i], RHS) ) \
+ out[i] = 3; \
+ }
+
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 0) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (MIN, _Float16, uint64_t, 32)
+
+TEST_ALL (MIN, _Float16, uint32_t, 64)
+
+TEST_ALL (MIN, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_2.c
new file mode 100644
index 0000000..81f583b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -fno-trapping-math -moverride=sve_width=2048" } */
+
+#include "unpacked_fminnm_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_1.c
new file mode 100644
index 0000000..312bccc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF) __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF) __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT) \
+ void \
+ f_##TYPE0##_##TYPE1 (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE0 *__restrict d) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN > d[i]) \
+ out[i] = 3; \
+ }
+
+TEST_FN (FMLA (f16), _Float16, uint64_t, 32)
+
+TEST_FN (FMLA (f16), _Float16, uint32_t, 64)
+
+TEST_FN (FMLA (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fmla|fmad)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fmla|fmad)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_2.c
new file mode 100644
index 0000000..ca3f94d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fmla_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fmla|fmad)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fmla|fmad)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_1.c
new file mode 100644
index 0000000..f7cbfb3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF) __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF) __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT) \
+ void \
+ f_##TYPE0##_##TYPE1 (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE0 *__restrict d) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN > d[i]) \
+ out[i] = 3; \
+ }
+
+TEST_FN (FMLS (f16), _Float16, uint64_t, 32)
+
+TEST_FN (FMLS (f16), _Float16, uint32_t, 64)
+
+TEST_FN (FMLS (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fmls|fmsb)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fmls|fmsb)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_2.c
new file mode 100644
index 0000000..387dbec
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fmls_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fmls|fmsb)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fmls|fmsb)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_1.c
new file mode 100644
index 0000000..a180a07
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_1.c
@@ -0,0 +1,43 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define immp5 0.5
+#define MUL(A, B) A * B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i], (TYPE0)RHS) > c[i]) \
+ out[i] = 3; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, immp5)
+
+TEST_ALL (MUL, _Float16, uint64_t, 32)
+
+TEST_ALL (MUL, _Float16, uint32_t, 64)
+
+TEST_ALL (MUL, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 5 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 5 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
new file mode 100644
index 0000000..eb05600
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fmul_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 5 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 5 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fneg_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fneg_1.c
new file mode 100644
index 0000000..d489ecb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fneg_1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define NEG(X) -X
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i]) > b[i]) \
+ out[i] = 3; \
+ }
+
+TEST_FN (NEG, _Float16, uint64_t, 32)
+
+TEST_FN (NEG, _Float16, uint32_t, 64)
+
+TEST_FN (NEG, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_1.c
new file mode 100644
index 0000000..bf13ff5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF) __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF) __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT) \
+ void \
+ f_##TYPE0##_##TYPE1 (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE0 *__restrict d) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN > d[i]) \
+ out[i] = 3; \
+ }
+
+TEST_FN (FNMLA (f16), _Float16, uint64_t, 32)
+
+TEST_FN (FNMLA (f16), _Float16, uint32_t, 64)
+
+TEST_FN (FNMLA (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fnmla|fnmad)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fnmla|fnmad)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_2.c
new file mode 100644
index 0000000..64130ba
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fnmla_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fnmla|fnmad)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fnmla|fnmad)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_1.c
new file mode 100644
index 0000000..399920a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF) __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF) __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT) \
+ void \
+ f_##TYPE0##_##TYPE1 (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c, \
+ TYPE0 *__restrict d) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN > d[i]) \
+ out[i] = 3; \
+ }
+
+TEST_FN (FNMLS (f16), _Float16, uint64_t, 32)
+
+TEST_FN (FNMLS (f16), _Float16, uint32_t, 64)
+
+TEST_FN (FNMLS (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fnmls|fnmsb)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fnmls|fnmsb)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_2.c
new file mode 100644
index 0000000..59fb7f9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fnmls_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fnmls|fnmsb)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fnmls|fnmsb)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_1.c
new file mode 100644
index 0000000..3cbdef3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i]) > b[i]) \
+ out[i] = 3; \
+ }
+
+TEST_FN (__builtin_roundf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_roundf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_roundf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_2.c
new file mode 100644
index 0000000..4564686
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frinta_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_1.c
new file mode 100644
index 0000000..7645fed
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i]) > b[i]) \
+ out[i] = 3; \
+ }
+
+TEST_FN (__builtin_nearbyintf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_nearbyintf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_nearbyintf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_2.c
new file mode 100644
index 0000000..eadce07
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frinti_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_1.c
new file mode 100644
index 0000000..98f85fb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i]) > b[i]) \
+ out[i] = 3; \
+ }
+
+TEST_FN (__builtin_floorf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_floorf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_floorf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_2.c
new file mode 100644
index 0000000..56988be
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frintm_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_1.c
new file mode 100644
index 0000000..f233697
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i]) > b[i]) \
+ out[i] = 3; \
+ }
+
+TEST_FN (__builtin_ceilf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_ceilf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_ceilf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_2.c
new file mode 100644
index 0000000..c24c632
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frintp_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_1.c
new file mode 100644
index 0000000..73403a5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i]) > b[i]) \
+ out[i] = 3; \
+ }
+
+TEST_FN (__builtin_rintf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_rintf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_rintf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_2.c
new file mode 100644
index 0000000..e8b8924
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frintx_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_1.c
new file mode 100644
index 0000000..7377843
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i]) > b[i]) \
+ out[i] = 3; \
+ }
+
+TEST_FN (__builtin_truncf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_truncf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_truncf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_2.c
new file mode 100644
index 0000000..1779122
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frintz_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_1.c
new file mode 100644
index 0000000..2cc8ec2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_1.c
@@ -0,0 +1,46 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define immp5 0.5
+#define SUBR(A, B) B - A
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS) \
+ void \
+ f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out, \
+ TYPE0 *__restrict a, \
+ TYPE0 *__restrict b, \
+ TYPE0 *__restrict c) \
+ { \
+ for (unsigned int i = 0; i < COUNT; i++) \
+ if (FN (a[i], (TYPE0)RHS) > c[i]) \
+ out[i] = 3; \
+ }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, immp5) \
+ TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (SUBR, _Float16, uint64_t, 32)
+
+TEST_ALL (SUBR, _Float16, uint32_t, 64)
+
+TEST_ALL (SUBR, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfsubr?\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfsubr?\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
new file mode 100644
index 0000000..de9325c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile }*/
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fsubr_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr119795.c b/gcc/testsuite/gcc.target/i386/pr119795.c
new file mode 100644
index 0000000..03c91cc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr119795.c
@@ -0,0 +1,26 @@
+/* { dg-do run } */
+/* { dg-options "-O -fschedule-insns -favoid-store-forwarding" } */
+
+unsigned a, b, c;
+
+void
+foo (_BitInt(2) b2, unsigned _BitInt(255) by, unsigned _BitInt(5) b5,
+ unsigned _BitInt(256) *ret)
+{
+ unsigned _BitInt(255) bx = b2;
+ by += 0x80000000000000000000000000000000wb;
+ __builtin_memmove (&b, &c, 3);
+ unsigned d = b;
+ unsigned e = __builtin_stdc_rotate_right (0x1uwb % b5, a);
+ unsigned _BitInt(256) r = by + bx + d + e;
+ *ret = r;
+}
+
+int
+main ()
+{
+ unsigned _BitInt(256) x;
+ foo (0, -1, 2, &x);
+ if (x != 0x80000000000000000000000000000000wb)
+ __builtin_abort();
+} \ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/i386/pr120427-5.c b/gcc/testsuite/gcc.target/i386/pr120427-5.c
new file mode 100644
index 0000000..7199aef
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120427-5.c
@@ -0,0 +1,10 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-Oz" } */
+
+long long
+func1 (void)
+{
+ return -1;
+}
+/* { dg-final { scan-assembler-times "pushq\[ \\t\]+\\\$-1" 1 } } */
+/* { dg-final { scan-assembler-times "popq\[ \\t\]+%rax" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-1a.c b/gcc/testsuite/gcc.target/i386/pr121208-1a.c
new file mode 100644
index 0000000..cb8bd0b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-1a.c
@@ -0,0 +1,15 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mno-80387 -mtls-dialect=gnu" } */
+
+extern __thread int bar;
+extern void func (void);
+
+__attribute__((no_caller_saved_registers))
+void
+foo (int error)
+{
+ bar = 1; /* { dg-error -mtls-dialect=gnu2 } */
+ if (error == 0)
+ func ();
+ bar = 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-1b.c b/gcc/testsuite/gcc.target/i386/pr121208-1b.c
new file mode 100644
index 0000000..037e9a0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-1b.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mno-80387 -mtls-dialect=gnu2" } */
+
+#include "pr121208-1a.c"
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-2a.c b/gcc/testsuite/gcc.target/i386/pr121208-2a.c
new file mode 100644
index 0000000..c1891ae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-2a.c
@@ -0,0 +1,17 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mtls-dialect=gnu" } */
+
+typedef unsigned int uword_t __attribute__ ((mode (__word__)));
+extern __thread int bar;
+extern void func (void);
+
+__attribute__((target("general-regs-only")))
+__attribute__((interrupt))
+void
+foo (void *frame, uword_t error)
+{
+ bar = 1; /* { dg-error -mtls-dialect=gnu2 } */
+ if (error == 0)
+ func ();
+ bar = 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-2b.c b/gcc/testsuite/gcc.target/i386/pr121208-2b.c
new file mode 100644
index 0000000..269b120
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-2b.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mtls-dialect=gnu2" } */
+
+#include "pr121208-2a.c"
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-3a.c b/gcc/testsuite/gcc.target/i386/pr121208-3a.c
new file mode 100644
index 0000000..26fe687
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-3a.c
@@ -0,0 +1,17 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mtls-dialect=gnu" } */
+
+typedef unsigned int uword_t __attribute__ ((mode (__word__)));
+extern __thread int bar;
+extern void func (void);
+
+__attribute__((target("general-regs-only")))
+__attribute__((interrupt))
+void
+foo (void *frame)
+{
+ bar = 1; /* { dg-error -mtls-dialect=gnu2 } */
+ if (frame == 0)
+ func ();
+ bar = 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-3b.c b/gcc/testsuite/gcc.target/i386/pr121208-3b.c
new file mode 100644
index 0000000..b672d75
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-3b.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mtls-dialect=gnu2" } */
+
+#include "pr121208-3a.c"
diff --git a/gcc/testsuite/gcc.target/i386/pr121274.c b/gcc/testsuite/gcc.target/i386/pr121274.c
new file mode 100644
index 0000000..16760cf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121274.c
@@ -0,0 +1,24 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-march=x86-64-v4 -O2" } */
+/* { dg-final { scan-assembler-not "vpextrq" } } */
+/* { dg-final { scan-assembler-not "vpinsrq" } } */
+
+typedef int v16si __attribute__((vector_size(64)));
+typedef int v4si __attribute__((vector_size(16)));
+
+v4si f(v16si x)
+{
+ return __builtin_shufflevector(x, x, 0, 1, 2, 3);
+}
+
+v4si g(v16si x)
+{
+return __builtin_shufflevector(x, x, 4, 5, 6, 7);
+}
+
+v4si f1(__int128 *x)
+{
+ __int128 t = *x;
+ asm("":"+x"(t));
+ return (v4si)t;
+}
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_100.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100.c
new file mode 100644
index 0000000..e759a11
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_100 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_100a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100a.c
new file mode 100644
index 0000000..153ed1e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_100a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_100f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100f.c
new file mode 100644
index 0000000..9bb9127
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_100f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_101.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101.c
new file mode 100644
index 0000000..06b3ceb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_101 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_101a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101a.c
new file mode 100644
index 0000000..0cca3f3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_101a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_101f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101f.c
new file mode 100644
index 0000000..9548be5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_101f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_103.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103.c
new file mode 100644
index 0000000..5731249
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_103 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_103a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103a.c
new file mode 100644
index 0000000..aea501e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_103a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_103f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103f.c
new file mode 100644
index 0000000..59d8987
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_103f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_120.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120.c
new file mode 100644
index 0000000..d28a671
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_120 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_120a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120a.c
new file mode 100644
index 0000000..613dd65
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_120a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_120f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120f.c
new file mode 100644
index 0000000..1b23350
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_120f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_121.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121.c
new file mode 100644
index 0000000..240332b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_121 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_121a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121a.c
new file mode 100644
index 0000000..1e7fb70
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_121a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_121f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121f.c
new file mode 100644
index 0000000..2cbec51
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_121f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^ \.version 7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^ \.target sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c b/gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c
index 5095d50..312043b 100644
--- a/gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c
+++ b/gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c
@@ -1,8 +1,16 @@
/* { dg-do compile { target { powerpc*-*-* } } } */
/* { dg-skip-if "" { powerpc*-*-darwin* } } */
-/* { dg-options "-O2 -mdejagnu-cpu=power7 -fno-inline-functions" } */
+/* { dg-options "-O2 -mdejagnu-cpu=power7 -fno-inline-functions -fno-ipa-icf" } */
/* { dg-require-effective-target powerpc_vsx } */
+/* PR testsuite/119382
+ Note: Added -fno-ipa-icf to disable Interprocedural Identical Code
+ Folding (ICF). Without this, insert_di_0_v2 is merged with insert_di_0
+ due to improved alias analysis introduced in commit r15-7961-gdc47161c1f32c3.
+ This results in the compiler replacing insert_di_0_v2 with a tail call to
+ insert_di_0, altering expected test behavior. Disabling ICF ensures correct
+ execution of the test. */
+
/* Test simple extract/insert/slat operations. Make sure all types are
supported with various options. */
diff --git a/gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c b/gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c
index 81ebf5f..15cc3ee 100644
--- a/gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c
+++ b/gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c
@@ -1,7 +1,7 @@
/* Verify proper errors are generated for conflicted interrupt type. */
/* { dg-do compile } */
/* { dg-options "" } */
-void __attribute__ ((interrupt ("user")))
+void __attribute__ ((interrupt ("supervisor")))
foo(void);
void __attribute__ ((interrupt ("machine")))
diff --git a/gcc/testsuite/gcc.target/riscv/interrupt-rnmi.c b/gcc/testsuite/gcc.target/riscv/interrupt-rnmi.c
new file mode 100644
index 0000000..f340108
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/interrupt-rnmi.c
@@ -0,0 +1,11 @@
+/* Verify the return instruction is mnret. */
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc_smrnmi" { target { rv32 } } } */
+/* { dg-options "-march=rv64gc_smrnmi" { target { rv64 } } } */
+
+void __attribute__ ((interrupt ("rnmi")))
+foo (void)
+{
+}
+
+/* { dg-final { scan-assembler {\mmnret} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/interrupt-umode.c b/gcc/testsuite/gcc.target/riscv/interrupt-umode.c
deleted file mode 100644
index 042abf0..0000000
--- a/gcc/testsuite/gcc.target/riscv/interrupt-umode.c
+++ /dev/null
@@ -1,8 +0,0 @@
-/* Verify the return instruction is mret. */
-/* { dg-do compile } */
-/* { dg-options "" } */
-void __attribute__ ((interrupt ("user")))
-foo (void)
-{
-}
-/* { dg-final { scan-assembler {\muret} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c
index 4dc5703..0fa1ea0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c
@@ -72,7 +72,7 @@ f_vnx128qi (int8_t *out)
*(vnx128qi *) out = v;
}
-/* { dg-final { scan-assembler-times {vmv.v.x\tv[0-9]+,\s*[a-x0-9]+} 6 } } */
+/* { dg-final { scan-assembler-times {vmv.v.x\tv[0-9]+,\s*[a-x0-9]+} 7 } } */
/* { dg-final { scan-assembler-times {slli\t[a-x0-9]+,\s*[a-x0-9]+,\s*8} 6 } } */
/* { dg-final { scan-assembler-times {or\t[a-x0-9]+,\s*[a-x0-9]+,\s*[a-x0-9]+} 6 } } */
/* { dg-final { scan-assembler-times {vslide1down\.vx\tv[0-9]+,\s*v[0-9]+,\s*[a-x0-9]+} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c
index 982dd97..fd8aa30 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c
@@ -1,5 +1,9 @@
/* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-march=rv64gcv_zvfh --param=fpr2vr-cost=0" } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
#include "vf_mulop.h"
#include "vf_mulop_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c
index 400bbcd..8fd8552 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c
@@ -1,5 +1,9 @@
/* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-march=rv64gcv_zvfh --param=fpr2vr-cost=0" } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
#include "vf_mulop.h"
#include "vf_mulop_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c
index 21c1860..e91fd15 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c
@@ -1,5 +1,9 @@
/* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-march=rv64gcv_zvfh --param=fpr2vr-cost=0" } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
#include "vf_mulop.h"
#include "vf_mulop_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c
index 163b5bd..ca7e0db 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c
@@ -1,5 +1,9 @@
/* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-march=rv64gcv_zvfh --param=fpr2vr-cost=0" } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
#include "vf_mulop.h"
#include "vf_mulop_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c
index 71f350f..b38e800 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c
@@ -1,5 +1,9 @@
/* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-march=rv64gcv_zvfh --param=fpr2vr-cost=0" } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
#include "vf_mulop.h"
#include "vf_mulop_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c
index e252e0d..fef5d77 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c
@@ -1,5 +1,9 @@
/* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-march=rv64gcv_zvfh --param=fpr2vr-cost=0" } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
#include "vf_mulop.h"
#include "vf_mulop_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c
index 439fd3e..7951d40 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c
@@ -1,5 +1,9 @@
/* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-march=rv64gcv_zvfh --param=fpr2vr-cost=0" } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
#include "vf_mulop.h"
#include "vf_mulop_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c
index b9d66ba..d0def86 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c
@@ -1,5 +1,9 @@
/* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-march=rv64gcv_zvfh --param=fpr2vr-cost=0" } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
#include "vf_mulop.h"
#include "vf_mulop_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c
index d78cf73..d4c527a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c
@@ -1,5 +1,9 @@
/* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-march=rv64gcv_zvfh --param=fpr2vr-cost=0" } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
#include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c
index 6422bba..abce2f2 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c
@@ -1,5 +1,9 @@
/* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-march=rv64gcv_zvfh --param=fpr2vr-cost=0" } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
#include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c
index 6be7d72..ddf49d5 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c
@@ -1,5 +1,9 @@
/* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-march=rv64gcv_zvfh --param=fpr2vr-cost=0" } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
#include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c
index dd28234..a874991 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c
@@ -1,5 +1,9 @@
/* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-march=rv64gcv_zvfh --param=fpr2vr-cost=0" } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
#include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c
index 83515ee..f84d7f5 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c
@@ -20,3 +20,4 @@ TEST_BINARY_VX_SIGNED_0(T)
/* { dg-final { scan-assembler-times {vmin.vx} 2 } } */
/* { dg-final { scan-assembler-times {vsadd.vx} 1 } } */
/* { dg-final { scan-assembler-times {vssub.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c
index 1488fe1..70b6743 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c
@@ -20,3 +20,4 @@ TEST_BINARY_VX_SIGNED_0(T)
/* { dg-final { scan-assembler-times {vmin.vx} 2 } } */
/* { dg-final { scan-assembler-times {vsadd.vx} 1 } } */
/* { dg-final { scan-assembler-times {vssub.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c
index 342ea18..986fa4c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c
@@ -20,3 +20,4 @@ TEST_BINARY_VX_SIGNED_0(T)
/* { dg-final { scan-assembler-times {vmin.vx} 2 } } */
/* { dg-final { scan-assembler-times {vsadd.vx} 1 } } */
/* { dg-final { scan-assembler-times {vssub.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 1 { target { no-opts "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2" "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4" } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c
index 583f917..c479295 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c
@@ -20,3 +20,4 @@ TEST_BINARY_VX_SIGNED_0(T)
/* { dg-final { scan-assembler-times {vmin.vx} 2 } } */
/* { dg-final { scan-assembler-times {vsadd.vx} 1 } } */
/* { dg-final { scan-assembler-times {vssub.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c
index e7b1ef0..8e7a788 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c
@@ -19,4 +19,7 @@ TEST_BINARY_VX_UNSIGNED_0(T)
/* { dg-final { scan-assembler-times {vminu.vx} 2 } } */
/* { dg-final { scan-assembler-times {vsaddu.vx} 1 } } */
/* { dg-final { scan-assembler-times {vssubu.vx} 1 } } */
-/* { dg-final { scan-assembler-times {vaaddu.vx} 1 { target { no-opts "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2" "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4" } } } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 2 { target { no-opts {
+ "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+ "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+ } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c
index 559887e..d213c18 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c
@@ -19,4 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
/* { dg-final { scan-assembler-times {vminu.vx} 2 } } */
/* { dg-final { scan-assembler-times {vsaddu.vx} 1 } } */
/* { dg-final { scan-assembler-times {vssubu.vx} 1 } } */
-/* { dg-final { scan-assembler-times {vaaddu.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 2 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c
index 78d3e0b..05801a9 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c
@@ -20,3 +20,4 @@ TEST_BINARY_VX_SIGNED_0(T)
/* { dg-final { scan-assembler-not {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c
index e7bcfe5..f05f091 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c
@@ -20,3 +20,4 @@ TEST_BINARY_VX_SIGNED_0(T)
/* { dg-final { scan-assembler-not {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c
index f9f1e39..adf9ccb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c
@@ -20,3 +20,4 @@ TEST_BINARY_VX_SIGNED_0(T)
/* { dg-final { scan-assembler-not {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c
index 80d6aaa..8b3f5bc 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c
@@ -20,3 +20,4 @@ TEST_BINARY_VX_SIGNED_0(T)
/* { dg-final { scan-assembler-not {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c
index 6bf2a35..741f431 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c
@@ -20,3 +20,4 @@ TEST_BINARY_VX_SIGNED_0(T)
/* { dg-final { scan-assembler-not {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c
index 5432706..1741c22 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c
@@ -20,3 +20,4 @@ TEST_BINARY_VX_SIGNED_0(T)
/* { dg-final { scan-assembler-not {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c
index a2099fd..d326357 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c
@@ -20,3 +20,4 @@ TEST_BINARY_VX_SIGNED_0(T)
/* { dg-final { scan-assembler-not {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c
index 1daede9..3137dc0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c
@@ -20,3 +20,4 @@ TEST_BINARY_VX_SIGNED_0(T)
/* { dg-final { scan-assembler-not {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c
index d79a9f2..86c8040 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -34,3 +35,9 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
/* { dg-final { scan-assembler {vmin.vx} } } */
/* { dg-final { scan-assembler {vsadd.vx} } } */
/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+ "-mrvv-vector-bits=scalable -mrvv-max-lmul=m1"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+ } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c
index 940f596..e2d1613 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -34,3 +35,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
/* { dg-final { scan-assembler {vmin.vx} } } */
/* { dg-final { scan-assembler {vsadd.vx} } } */
/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+ } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c
index 22a64f6..06ffa15 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -34,3 +35,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
/* { dg-final { scan-assembler {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { no-opts "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2" "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4" } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c
index 3286b1a..cb086aa 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -34,3 +35,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
/* { dg-final { scan-assembler {vmin.vx} } } */
/* { dg-final { scan-assembler {vsadd.vx} } } */
/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+ } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c
index c851f23..3ecfce6 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c
index b7805c1..7ce1fe8 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -31,5 +32,6 @@ DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_B
/* { dg-final { scan-assembler {vremu.vx} } } */
/* { dg-final { scan-assembler {vmaxu.vx} } } */
/* { dg-final { scan-assembler {vminu.vx} } } */
+/* { dg-final { scan-assembler {vsaddu.vx} } } */
/* { dg-final { scan-assembler {vssubu.vx} } } */
/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c
index 8295dc2..c84a30c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -33,4 +34,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_B
/* { dg-final { scan-assembler {vminu.vx} } } */
/* { dg-final { scan-assembler-not {vsaddu.vx} } } */
/* { dg-final { scan-assembler-not {vssubu.vx} } } */
-/* { dg-final { scan-assembler {vaaddu.vx} { target { no-opts "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2" "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4" } } } } */
+/* { dg-final { scan-assembler {vaaddu.vx} { target { no-opts {
+ "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+ "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+ } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c
index d214da9..9f3d7df 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c
index b92db10..7c7bf09 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
/* { dg-final { scan-assembler-not {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -34,3 +35,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
/* { dg-final { scan-assembler {vmin.vx} } } */
/* { dg-final { scan-assembler {vsadd.vx} } } */
/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+ } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c
index 0870cde..6d161bd 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -34,3 +35,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
/* { dg-final { scan-assembler {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { no-opts {
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+ } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c
index a4d60e9..0409012 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -34,3 +35,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
/* { dg-final { scan-assembler {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c
index ec069a3..ed437319 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
/* { dg-final { scan-assembler-not {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -34,3 +35,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
/* { dg-final { scan-assembler {vmin.vx} } } */
/* { dg-final { scan-assembler {vsadd.vx} } } */
/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { no-opts {
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+ } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c
index b7c7ad4..5497b5a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c
index dd9c845..3a8e85f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c
index 1fda062..060d591 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c
index ee6d6aa..86a6c45 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c
@@ -19,7 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
-DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -33,4 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_B
/* { dg-final { scan-assembler {vminu.vx} } } */
/* { dg-final { scan-assembler {vsaddu.vx} } } */
/* { dg-final { scan-assembler {vssubu.vx} } } */
-/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c
index 473c31b..1e18342 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
/* { dg-final { scan-assembler-not {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -34,3 +35,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
/* { dg-final { scan-assembler {vmin.vx} } } */
/* { dg-final { scan-assembler {vsadd.vx} } } */
/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+ } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c
index 6ae84c1..fd6e47c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -34,3 +35,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
/* { dg-final { scan-assembler {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+ } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c
index 794f506..399d0f5 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
/* { dg-final { scan-assembler-not {vadd.vx} } } */
/* { dg-final { scan-assembler-not {vsub.vx} } } */
@@ -34,3 +35,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
/* { dg-final { scan-assembler-not {vmin.vx} } } */
/* { dg-final { scan-assembler-not {vsadd.vx} } } */
/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+ } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c
index 77bcdeb..98567a3 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
/* { dg-final { scan-assembler-not {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
@@ -34,3 +35,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
/* { dg-final { scan-assembler {vmin.vx} } } */
/* { dg-final { scan-assembler {vsadd.vx} } } */
/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+ "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+ } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c
index 3a215ea..f51e7a1 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c
index ac4d100..79b7477 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c
index 5eb0ed6..ac5fd69 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
/* { dg-final { scan-assembler-not {vadd.vx} } } */
/* { dg-final { scan-assembler-not {vsub.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c
index 8b404b6..84aa06b 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c
@@ -20,6 +20,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
/* { dg-final { scan-assembler {vadd.vx} } } */
/* { dg-final { scan-assembler {vsub.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i16.c
new file mode 100644
index 0000000..2b87321
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i16.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL 8
+
+#include "vx-fixed-vxrm.h"
+
+#define VT vint16m1_t
+#define T int16_t
+#define ELEM_SIZE 16
+#define SUFFIX i16
+#define FUNC __riscv_vaadd_vv_i16m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i32.c
new file mode 100644
index 0000000..b95699b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i32.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL 4
+
+#include "vx-fixed-vxrm.h"
+
+#define VT vint32m1_t
+#define T int32_t
+#define ELEM_SIZE 32
+#define SUFFIX i32
+#define FUNC __riscv_vaadd_vv_i32m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i64.c
new file mode 100644
index 0000000..48b6010
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i64.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL 2
+
+#include "vx-fixed-vxrm.h"
+
+#define VT vint64m1_t
+#define T int64_t
+#define ELEM_SIZE 64
+#define SUFFIX i64
+#define FUNC __riscv_vaadd_vv_i64m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i8.c
new file mode 100644
index 0000000..d07a625
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i8.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL 16
+
+#include "vx-fixed-vxrm.h"
+
+#define VT vint8m1_t
+#define T int8_t
+#define ELEM_SIZE 8
+#define SUFFIX i8
+#define FUNC __riscv_vaadd_vv_i8m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u16.c
new file mode 100644
index 0000000..bd36429
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u16.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL 8
+
+#include "vx-fixed-vxrm.h"
+
+#define VT vuint16m1_t
+#define T uint16_t
+#define ELEM_SIZE 16
+#define SUFFIX u16
+#define FUNC __riscv_vaaddu_vv_u16m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u32.c
new file mode 100644
index 0000000..f023a76
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u32.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL 4
+
+#include "vx-fixed-vxrm.h"
+
+#define VT vuint32m1_t
+#define T uint32_t
+#define ELEM_SIZE 32
+#define SUFFIX u32
+#define FUNC __riscv_vaaddu_vv_u32m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u64.c
new file mode 100644
index 0000000..d9a37ae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u64.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL 2
+
+#include "vx-fixed-vxrm.h"
+
+#define VT vuint64m1_t
+#define T uint64_t
+#define ELEM_SIZE 64
+#define SUFFIX u64
+#define FUNC __riscv_vaaddu_vv_u64m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u8.c
new file mode 100644
index 0000000..328e5d4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u8.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL 16
+
+#include "vx-fixed-vxrm.h"
+
+#define VT vuint8m1_t
+#define T uint8_t
+#define ELEM_SIZE 8
+#define SUFFIX u8
+#define FUNC __riscv_vaaddu_vv_u8m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm.h
new file mode 100644
index 0000000..438c7ab
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm.h
@@ -0,0 +1,28 @@
+#ifndef HAVE_DEFINED_VX_FIXED_VXRM_H
+#define HAVE_DEFINED_VX_FIXED_VXRM_H
+
+#include <riscv_vector.h>
+
+int64_t go[VL] = {};
+int64_t ga[VL] = {};
+
+#define DEF_FIXED_BINARY_VX(VT, T, ES, SX, VXRM, FUNC) \
+void __attribute__((noinline)) \
+test_fixed_binary_##VT##_##VXRM##_##FUNC##_vx () { \
+ VT a = __riscv_vle##ES##_v_##SX##m1((T *)ga, VL); \
+ VT b; \
+ T *bp = (T *)&b; \
+ \
+ for (int i = 0; i < VL; i++) { \
+ bp[i] = 123; \
+ } \
+ \
+ VT d = FUNC (a, b, VXRM, VL); \
+ \
+ __riscv_vse##ES##_v_##SX##m1((T *)&go, d, VL); \
+}
+
+#define DEF_FIXED_BINARY_VX_WRAP(VT, T, ES, SX, VXRM, FUNC) \
+ DEF_FIXED_BINARY_VX(VT, T, ES, SX, VXRM, FUNC)
+
+#endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h
index 38f2f72..de48ebd 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h
@@ -3,11 +3,12 @@
#include <stdint.h>
-#undef HAS_UINT128
+#undef HAS_INT128
#if __riscv_xlen == 64
-#define HAS_UINT128
+#define HAS_INT128
typedef unsigned __int128 uint128_t;
+typedef signed __int128 int128_t;
#endif
#define DEF_VX_BINARY_CASE_0(T, OP, NAME) \
@@ -358,29 +359,51 @@ DEF_AVG_FLOOR(uint8_t, uint16_t)
DEF_AVG_FLOOR(uint16_t, uint32_t)
DEF_AVG_FLOOR(uint32_t, uint64_t)
-#ifdef HAS_UINT128
+DEF_AVG_FLOOR(int8_t, int16_t)
+DEF_AVG_FLOOR(int16_t, int32_t)
+DEF_AVG_FLOOR(int32_t, int64_t)
+
+#define DEF_AVG_CEIL(NT, WT) \
+NT \
+test_##NT##_avg_ceil(NT x, NT y) \
+{ \
+ return (NT)(((WT)x + (WT)y + 1) >> 1); \
+}
+
+DEF_AVG_CEIL(uint8_t, uint16_t)
+DEF_AVG_CEIL(uint16_t, uint32_t)
+DEF_AVG_CEIL(uint32_t, uint64_t)
+
+#ifdef HAS_INT128
DEF_AVG_FLOOR(uint64_t, uint128_t)
+ DEF_AVG_FLOOR(int64_t, int128_t)
+
+ DEF_AVG_CEIL(uint64_t, uint128_t)
#endif
#define AVG_FLOOR_FUNC(T) test_##T##_avg_floor
#define AVG_FLOOR_FUNC_WRAP(T) AVG_FLOOR_FUNC(T)
-#define TEST_BINARY_VX_SIGNED_0(T) \
- DEF_VX_BINARY_CASE_0_WRAP(T, +, add) \
- DEF_VX_BINARY_CASE_0_WRAP(T, -, sub) \
- DEF_VX_BINARY_REVERSE_CASE_0_WRAP(T, -, rsub) \
- DEF_VX_BINARY_CASE_0_WRAP(T, &, and) \
- DEF_VX_BINARY_CASE_0_WRAP(T, |, or) \
- DEF_VX_BINARY_CASE_0_WRAP(T, ^, xor) \
- DEF_VX_BINARY_CASE_0_WRAP(T, *, mul) \
- DEF_VX_BINARY_CASE_0_WRAP(T, /, div) \
- DEF_VX_BINARY_CASE_0_WRAP(T, %, rem) \
- DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_0_WARP(T), max) \
- DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_1_WARP(T), max) \
- DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_0_WARP(T), min) \
- DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_1_WARP(T), min) \
- DEF_VX_BINARY_CASE_2_WRAP(T, SAT_S_ADD_FUNC(T), sat_add) \
- DEF_VX_BINARY_CASE_2_WRAP(T, SAT_S_SUB_FUNC(T), sat_sub) \
+#define AVG_CEIL_FUNC(T) test_##T##_avg_ceil
+#define AVG_CEIL_FUNC_WRAP(T) AVG_CEIL_FUNC(T)
+
+#define TEST_BINARY_VX_SIGNED_0(T) \
+ DEF_VX_BINARY_CASE_0_WRAP(T, +, add) \
+ DEF_VX_BINARY_CASE_0_WRAP(T, -, sub) \
+ DEF_VX_BINARY_REVERSE_CASE_0_WRAP(T, -, rsub) \
+ DEF_VX_BINARY_CASE_0_WRAP(T, &, and) \
+ DEF_VX_BINARY_CASE_0_WRAP(T, |, or) \
+ DEF_VX_BINARY_CASE_0_WRAP(T, ^, xor) \
+ DEF_VX_BINARY_CASE_0_WRAP(T, *, mul) \
+ DEF_VX_BINARY_CASE_0_WRAP(T, /, div) \
+ DEF_VX_BINARY_CASE_0_WRAP(T, %, rem) \
+ DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_0_WARP(T), max) \
+ DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_1_WARP(T), max) \
+ DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_0_WARP(T), min) \
+ DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_1_WARP(T), min) \
+ DEF_VX_BINARY_CASE_2_WRAP(T, SAT_S_ADD_FUNC(T), sat_add) \
+ DEF_VX_BINARY_CASE_2_WRAP(T, SAT_S_SUB_FUNC(T), sat_sub) \
+ DEF_VX_BINARY_CASE_2_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor) \
#define TEST_BINARY_VX_UNSIGNED_0(T) \
DEF_VX_BINARY_CASE_0_WRAP(T, +, add) \
@@ -398,5 +421,6 @@ DEF_AVG_FLOOR(uint32_t, uint64_t)
DEF_VX_BINARY_CASE_2_WRAP(T, SAT_U_ADD_FUNC(T), sat_add) \
DEF_VX_BINARY_CASE_2_WRAP(T, SAT_U_SUB_FUNC(T), sat_sub) \
DEF_VX_BINARY_CASE_2_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor) \
+ DEF_VX_BINARY_CASE_2_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil) \
#endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h
index b742856..5024ae7 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h
@@ -5102,4 +5102,396 @@ uint64_t TEST_BINARY_DATA(uint64_t, avg_floor)[][3][N] =
},
};
+int8_t TEST_BINARY_DATA(int8_t, avg_floor)[][3][N] =
+{
+ {
+ { 0 },
+ {
+ 2, 2, 2, 2,
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 4, 4, 4, 4,
+ },
+ {
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 2, 2, 2, 2,
+ },
+ },
+ {
+ { 127 },
+ {
+ 127, 127, 127, 127,
+ -128, -128, -128, -128,
+ -127, -127, -127, -127,
+ 1, 1, 1, 1,
+ },
+ {
+ 127, 127, 127, 127,
+ -1, -1, -1, -1,
+ 0, 0, 0, 0,
+ 64, 64, 64, 64,
+ },
+ },
+ {
+ {-128 },
+ {
+ 0, 0, 0, 0,
+ -128, -128, -128, -128,
+ 126, 126, 126, 126,
+ 127, 127, 127, 127,
+ },
+ {
+ -64, -64, -64, -64,
+ -128, -128, -128, -128,
+ -1, -1, -1, -1,
+ -1, -1, -1, -1,
+ },
+ },
+};
+
+int16_t TEST_BINARY_DATA(int16_t, avg_floor)[][3][N] =
+{
+ {
+ { 0 },
+ {
+ 2, 2, 2, 2,
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 4, 4, 4, 4,
+ },
+ {
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 2, 2, 2, 2,
+ },
+ },
+ {
+ { 32767 },
+ {
+ 32767, 32767, 32767, 32767,
+ -32768, -32768, -32768, -32768,
+ -32767, -32767, -32767, -32767,
+ 1, 1, 1, 1,
+ },
+ {
+ 32767, 32767, 32767, 32767,
+ -1, -1, -1, -1,
+ 0, 0, 0, 0,
+ 16384, 16384, 16384, 16384,
+ },
+ },
+ {
+ {-32768 },
+ {
+ 0, 0, 0, 0,
+ -32768, -32768, -32768, -32768,
+ 32766, 32766, 32766, 32766,
+ 32767, 32767, 32767, 32767,
+ },
+ {
+ -16384, -16384, -16384, -16384,
+ -32768, -32768, -32768, -32768,
+ -1, -1, -1, -1,
+ -1, -1, -1, -1,
+ },
+ },
+};
+
+int32_t TEST_BINARY_DATA(int32_t, avg_floor)[][3][N] =
+{
+ {
+ { 0 },
+ {
+ 2, 2, 2, 2,
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 4, 4, 4, 4,
+ },
+ {
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 2, 2, 2, 2,
+ },
+ },
+ {
+ { 2147483647 },
+ {
+ 2147483647, 2147483647, 2147483647, 2147483647,
+ -2147483648, -2147483648, -2147483648, -2147483648,
+ -2147483647, -2147483647, -2147483647, -2147483647,
+ 1, 1, 1, 1,
+ },
+ {
+ 2147483647, 2147483647, 2147483647, 2147483647,
+ -1, -1, -1, -1,
+ 0, 0, 0, 0,
+ 1073741824, 1073741824, 1073741824, 1073741824,
+ },
+ },
+ {
+ {-2147483648 },
+ {
+ 0, 0, 0, 0,
+ -2147483648, -2147483648, -2147483648, -2147483648,
+ 2147483646, 2147483646, 2147483646, 2147483646,
+ 2147483647, 2147483647, 2147483647, 2147483647,
+ },
+ {
+ -1073741824, -1073741824, -1073741824, -1073741824,
+ -2147483648, -2147483648, -2147483648, -2147483648,
+ -1, -1, -1, -1,
+ -1, -1, -1, -1,
+ },
+ },
+};
+
+int64_t TEST_BINARY_DATA(int64_t, avg_floor)[][3][N] =
+{
+ {
+ { 0 },
+ {
+ 2, 2, 2, 2,
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 4, 4, 4, 4,
+ },
+ {
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 2, 2, 2, 2,
+ },
+ },
+ {
+ { 9223372036854775807ull },
+ {
+ 9223372036854775807ull, 9223372036854775807ull, 9223372036854775807ull, 9223372036854775807ull,
+ -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+ -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
+ 1, 1, 1, 1,
+ },
+ {
+ 9223372036854775807ull, 9223372036854775807ull, 9223372036854775807ull, 9223372036854775807ull,
+ -1, -1, -1, -1,
+ 0, 0, 0, 0,
+ 4611686018427387904ull, 4611686018427387904ull, 4611686018427387904ull, 4611686018427387904ull,
+ },
+ },
+ {
+ {-9223372036854775808ull },
+ {
+ 0, 0, 0, 0,
+ -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+ 9223372036854775807ull, 9223372036854775807ull, 9223372036854775807ull, 9223372036854775807ull,
+ 9223372036854775807ull, 9223372036854775807ull, 9223372036854775807ull, 9223372036854775807ull,
+ },
+ {
+ -4611686018427387904ull, -4611686018427387904ull, -4611686018427387904ull, -4611686018427387904ull,
+ -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+ -1, -1, -1, -1,
+ -1, -1, -1, -1,
+ },
+ },
+};
+
+uint8_t TEST_BINARY_DATA(uint8_t, avg_ceil)[][3][N] =
+{
+ {
+ { 0 },
+ {
+ 2, 2, 2, 2,
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 4, 4, 4, 4,
+ },
+ {
+ 1, 1, 1, 1,
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 2, 2, 2, 2,
+ },
+ },
+ {
+ { 127 },
+ {
+ 127, 127, 127, 127,
+ 128, 128, 128, 128,
+ 255, 255, 255, 255,
+ 1, 1, 1, 1,
+ },
+ {
+ 127, 127, 127, 127,
+ 128, 128, 128, 128,
+ 191, 191, 191, 191,
+ 64, 64, 64, 64,
+ },
+ },
+ {
+ { 255 },
+ {
+ 0, 0, 0, 0,
+ 255, 255, 255, 255,
+ 254, 254, 254, 254,
+ 1, 1, 1, 1,
+ },
+ {
+ 128, 128, 128, 128,
+ 255, 255, 255, 255,
+ 255, 255, 255, 255,
+ 128, 128, 128, 128,
+ },
+ },
+};
+
+uint16_t TEST_BINARY_DATA(uint16_t, avg_ceil)[][3][N] =
+{
+ {
+ { 0 },
+ {
+ 2, 2, 2, 2,
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 4, 4, 4, 4,
+ },
+ {
+ 1, 1, 1, 1,
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 2, 2, 2, 2,
+ },
+ },
+ {
+ { 32767 },
+ {
+ 32767, 32767, 32767, 32767,
+ 32768, 32768, 32768, 32768,
+ 65535, 65535, 65535, 65535,
+ 1, 1, 1, 1,
+ },
+ {
+ 32767, 32767, 32767, 32767,
+ 32768, 32768, 32768, 32768,
+ 49151, 49151, 49151, 49151,
+ 16384, 16384, 16384, 16384,
+ },
+ },
+ {
+ { 65535 },
+ {
+ 0, 0, 0, 0,
+ 65535, 65535, 65535, 65535,
+ 65534, 65534, 65534, 65534,
+ 1, 1, 1, 1,
+ },
+ {
+ 32768, 32768, 32768, 32768,
+ 65535, 65535, 65535, 65535,
+ 65535, 65535, 65535, 65535,
+ 32768, 32768, 32768, 32768,
+ },
+ },
+};
+
+uint32_t TEST_BINARY_DATA(uint32_t, avg_ceil)[][3][N] =
+{
+ {
+ { 0 },
+ {
+ 2, 2, 2, 2,
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 4, 4, 4, 4,
+ },
+ {
+ 1, 1, 1, 1,
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 2, 2, 2, 2,
+ },
+ },
+ {
+ { 2147483647 },
+ {
+ 2147483647, 2147483647, 2147483647, 2147483647,
+ 2147483648, 2147483648, 2147483648, 2147483648,
+ 4294967295, 4294967295, 4294967295, 4294967295,
+ 1, 1, 1, 1,
+ },
+ {
+ 2147483647, 2147483647, 2147483647, 2147483647,
+ 2147483648, 2147483648, 2147483648, 2147483648,
+ 3221225471, 3221225471, 3221225471, 3221225471,
+ 1073741824, 1073741824, 1073741824, 1073741824,
+ },
+ },
+ {
+ { 4294967295 },
+ {
+ 0, 0, 0, 0,
+ 4294967295, 4294967295, 4294967295, 4294967295,
+ 4294967294, 4294967294, 4294967294, 4294967294,
+ 1, 1, 1, 1,
+ },
+ {
+ 2147483648, 2147483648, 2147483648, 2147483648,
+ 4294967295, 4294967295, 4294967295, 4294967295,
+ 4294967295, 4294967295, 4294967295, 4294967295,
+ 2147483648, 2147483648, 2147483648, 2147483648,
+ },
+ },
+};
+
+uint64_t TEST_BINARY_DATA(uint64_t, avg_ceil)[][3][N] =
+{
+ {
+ { 0 },
+ {
+ 2, 2, 2, 2,
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 4, 4, 4, 4,
+ },
+ {
+ 1, 1, 1, 1,
+ 1, 1, 1, 1,
+ 0, 0, 0, 0,
+ 2, 2, 2, 2,
+ },
+ },
+ {
+ { 9223372036854775807ull },
+ {
+ 9223372036854775807ull, 9223372036854775807ull, 9223372036854775807ull, 9223372036854775807ull,
+ 9223372036854775808ull, 9223372036854775808ull, 9223372036854775808ull, 9223372036854775808ull,
+ 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull,
+ 1, 1, 1, 1,
+ },
+ {
+ 9223372036854775807ull, 9223372036854775807ull, 9223372036854775807ull, 9223372036854775807ull,
+ 9223372036854775808ull, 9223372036854775808ull, 9223372036854775808ull, 9223372036854775808ull,
+ 13835058055282163711ull, 13835058055282163711ull, 13835058055282163711ull, 13835058055282163711ull,
+ 4611686018427387904ull, 4611686018427387904ull, 4611686018427387904ull, 4611686018427387904ull,
+ },
+ },
+ {
+ { 18446744073709551615ull },
+ {
+ 0, 0, 0, 0,
+ 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull,
+ 18446744073709551614ull, 18446744073709551614ull, 18446744073709551614ull, 18446744073709551614ull,
+ 1, 1, 1, 1,
+ },
+ {
+ 9223372036854775808ull, 9223372036854775808ull, 9223372036854775808ull, 9223372036854775808ull,
+ 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull,
+ 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull,
+ 9223372036854775808ull, 9223372036854775808ull, 9223372036854775808ull, 9223372036854775808ull,
+ },
+ },
+};
+
#endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i16.c
new file mode 100644
index 0000000..0307b3f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T int16_t
+#define NAME avg_floor
+#define FUNC AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+ RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i32.c
new file mode 100644
index 0000000..d73325b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T int32_t
+#define NAME avg_floor
+#define FUNC AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+ RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i64.c
new file mode 100644
index 0000000..481774b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T int64_t
+#define NAME avg_floor
+#define FUNC AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+ RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i8.c
new file mode 100644
index 0000000..7de89ee
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T int8_t
+#define NAME avg_floor
+#define FUNC AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+ RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u16.c
new file mode 100644
index 0000000..6297672
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T uint16_t
+#define NAME avg_ceil
+#define FUNC AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+ RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u32.c
new file mode 100644
index 0000000..30db24b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T uint32_t
+#define NAME avg_ceil
+#define FUNC AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+ RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u64.c
new file mode 100644
index 0000000..db3c911
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T uint64_t
+#define NAME avg_ceil
+#define FUNC AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+ RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u8.c
new file mode 100644
index 0000000..a7755f0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T uint8_t
+#define NAME avg_ceil
+#define FUNC AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+ RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c
index 04dec7b..4f6785a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c
@@ -6,9 +6,9 @@
/*
** foo:
-** addi\t[a-x0-9]+,\s*[a-x0-9]+,100
+** ...
** vsetvli\tzero,a2,e64,m2,t[au],m[au]
-** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
+** vmv.s.x\tv[0-9]+.*
** vs2r.v\tv[0-9]+,0\([a-x0-9]+\)
** ret
*/
@@ -23,7 +23,7 @@ void foo (void *base, void *out, size_t vl)
** foo2:
** fld\tfa[0-9]+,\s*100\(a0\)
** vsetvli\tzero,a2,e64,m2,t[au],m[au]
-** vfmv\.v\.f\tv[0-9]+,\s*fa[0-9]+
+** vfmv\.s\.f\tv[0-9]+,\s*fa[0-9]+
** vs2r.v\tv[0-9]+,0\([a-x0-9]+\)
** ret
*/
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c
index 0ebb92e..a8c9263c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c
@@ -23,7 +23,7 @@ void foo (void *base, void *out, size_t vl)
** foo2:
** fld\tfa[0-9]+,\s*100\(a0\)
** vsetvli\tzero,a2,e64,m2,t[au],m[au]
-** vfmv\.v\.f\tv[0-9]+,\s*fa[0-9]+
+** vfmv\.s\.f\tv[0-9]+,\s*fa[0-9]+
** vs2r.v\tv[0-9]+,0\([a-x0-9]+\)
** ret
*/
@@ -52,7 +52,7 @@ void foo3 (void *base, void *out, size_t vl)
/*
** foo4:
** ...
-** vfmv\.v\.f\tv[0-9]+,\s*fa[0-9]+
+** vfmv\.s\.f\tv[0-9]+,\s*fa[0-9]+
** ...
** ret
*/
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c
index 512fa62..cf53aca 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c
@@ -6,9 +6,9 @@
/*
** foo:
-** addi\t[a-x0-9]+,\s*[a-x0-9]+,100
+** ...
** vsetvli\tzero,a2,e64,m2,t[au],m[au]
-** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
+** vmv\.v\.x\tv[0-9]+,\s*a[0-9]+
** vs2r.v\tv[0-9]+,0\([a-x0-9]+\)
** ret
*/
@@ -37,7 +37,7 @@ void foo2 (void *base, void *out, size_t vl)
/*
** foo3:
** ...
-** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
+** vmv\.v\.x\tv[0-9]+,\s*a[0-9]+
** ...
** ret
*/
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c
index d9d10f3..fd3b7c5 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c
@@ -175,9 +175,8 @@ void foo12 (void *base, void *out, size_t vl)
/*
** foo13:
** ...
-** vmv.v.x\tv[0-9]+,\s*[a-x0-9]+
+** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
** ...
-** ret
*/
void foo13 (void *base, void *out, size_t vl)
{
@@ -189,7 +188,7 @@ void foo13 (void *base, void *out, size_t vl)
/*
** foo14:
** ...
-** vmv.v.x\tv[0-9]+,\s*[a-x0-9]+
+** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
** ...
*/
void foo14 (void *base, void *out, size_t vl)
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c
index 80ee1b5..64c22dd 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c
@@ -23,4 +23,3 @@ vuint64m2_t f3(vuint64m2_t var_17, uint64_t var_60, size_t vl)
/* { dg-final { scan-assembler-times {vsetivli\s+zero,\s*0,\s*e64,\s*m2,\s*t[au],\s*m[au]} 1 } } */
/* { dg-final { scan-assembler-times {vsetivli\s+zero,\s*1,\s*e64,\s*m2,\s*t[au],\s*m[au]} 1 } } */
-/* { dg-final { scan-assembler-times {sgtu} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/pr121073.c b/gcc/testsuite/gcc.target/riscv/rvv/pr121073.c
new file mode 100644
index 0000000..2789d0f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/pr121073.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -O3 -mrvv-vector-bits=zvl -fsigned-char -fno-strict-aliasing -fwrapv -Wno-stringop-overflow -Wno-aggressive-loop-optimizations" } */
+
+int a;
+unsigned char p[1][21];
+void init() {
+ for (int s = 0; s < 21; ++s)
+ for (int t = 0; t < 21; ++t)
+ p[s][t] = 39;
+ for (short t = 0; t < 9; t += -5077966496202321318LL + 28071)
+ a = p[3][t] && p[2][t];
+}
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u32.c
new file mode 100644
index 0000000..7409232
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u32.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint16_t
+#define WT uint32_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u16.c
new file mode 100644
index 0000000..ec79e5d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u16.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint16_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u32.c
new file mode 100644
index 0000000..eb95184
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u32.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint32_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u16-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u16-from-u64.c
new file mode 100644
index 0000000..b1d33a9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u16-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint16_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u32-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u32-from-u64.c
new file mode 100644
index 0000000..af5ffecf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u32-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint32_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u8-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u8-from-u64.c
new file mode 100644
index 0000000..d65cab0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u8-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u32.c
new file mode 100644
index 0000000..e212391
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u32.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT uint16_t
+#define WT uint32_t
+#define NAME usmul
+#define DATA TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c
index 065afb8..79d3fb3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { rv32 } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
/* { dg-additional-options "-std=c99" } */
#include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c
index 062bbc9..ad63db3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { rv32 } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
/* { dg-additional-options "-std=c99" } */
#include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u16.c
new file mode 100644
index 0000000..f5a0ab5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u16.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT uint8_t
+#define WT uint16_t
+#define NAME usmul
+#define DATA TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u32.c
new file mode 100644
index 0000000..32074a4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u32.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT uint8_t
+#define WT uint32_t
+#define NAME usmul
+#define DATA TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c
index e6f632b..16ca905 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { rv32 } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
/* { dg-additional-options "-std=c99" } */
#include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c b/gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c
index dc5609c..167fa15 100644
--- a/gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c
+++ b/gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c
@@ -20,12 +20,6 @@ void func_machine (void)
/* { dg-final { scan-assembler-times {\mth\.ipop\M} 2 { target { rv32 } } } } */
-__attribute__ ((interrupt ("user")))
-void func_usr (void)
-{
- f ();
-}
-
__attribute__ ((interrupt ("supervisor")))
void func_supervisor (void)
{
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-fp-1.c b/gcc/testsuite/gcc.target/s390/spaceship-fp-1.c
new file mode 100644
index 0000000..56c3d77
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-fp-1.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 2\)} 3 optimized } } */
+/* { dg-final { scan-assembler-times {\tk[edx]br\t} 3 } } */
+/* { dg-final { scan-assembler-not {\tbrc} } } */
+/* { dg-final { scan-assembler-not {\tc[edx]br\t} } } */
+
+#define TEST(T, U) \
+ int test_##U (T x, T y) \
+ { \
+ if (x == y) \
+ return 0; \
+ else if (x < y) \
+ return -1; \
+ else if (x > y) \
+ return 1; \
+ else \
+ return 2; \
+ }
+
+TEST (float, float)
+TEST (double, double)
+TEST (long double, longdouble)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-fp-2.c b/gcc/testsuite/gcc.target/s390/spaceship-fp-2.c
new file mode 100644
index 0000000..0c6e6b6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-fp-2.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -ffinite-math-only -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 2\)} 3 optimized } } */
+/* { dg-final { scan-assembler-times {\tc[edx]br\t} 3 } } */
+/* { dg-final { scan-assembler-not {\tbrc} } } */
+/* { dg-final { scan-assembler-not {\tk[edx]br\t} } } */
+
+#define TEST(T, U) \
+ int test_##U (T x, T y) \
+ { \
+ if (x == y) \
+ return 0; \
+ else if (x < y) \
+ return -1; \
+ else if (x > y) \
+ return 1; \
+ else \
+ return 2; \
+ }
+
+TEST (float, float)
+TEST (double, double)
+TEST (long double, longdouble)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-fp-3.c b/gcc/testsuite/gcc.target/s390/spaceship-fp-3.c
new file mode 100644
index 0000000..2f567d1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-fp-3.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 42\)} 3 optimized } } */
+/* { dg-final { scan-assembler-times {\tk[edx]br\t} 3 } } */
+/* { dg-final { scan-assembler-not {\tbrc} } } */
+/* { dg-final { scan-assembler-not {\tc[edx]br\t} } } */
+
+#define TEST(T, U) \
+ int test_##U (T x, T y) \
+ { \
+ if (x == y) \
+ return 0; \
+ else if (x < y) \
+ return -1; \
+ else if (x > y) \
+ return 1; \
+ else \
+ return 42; \
+ }
+
+TEST (float, float)
+TEST (double, double)
+TEST (long double, longdouble)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-fp-4.c b/gcc/testsuite/gcc.target/s390/spaceship-fp-4.c
new file mode 100644
index 0000000..4531ecb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-fp-4.c
@@ -0,0 +1,53 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 0\)} 3 optimized } } */
+/* { dg-final { scan-assembler-times {\tk[edx]br\t} 3 } } */
+/* { dg-final { scan-assembler-not {\tloc} } } */
+/* { dg-final { scan-assembler-not {\tbrc} } } */
+/* { dg-final { scan-assembler-not {\tc[edx]br\t} } } */
+
+/* By time of writing this we emit
+
+ kebr %f0,%f2
+ jo .L2
+ je .L3
+ jnh .L10
+ jg f3@PLT
+.L10:
+ jg f2@PLT
+.L3:
+ jg f1@PLT
+.L2:
+ jg f4@PLT
+
+ which is not optimal. Instead we could fold the conditional branch with the
+ unconditional into something along the lines
+
+ kebr %f0,%f2
+ jo f4@PLT
+ je f1@PLT
+ jnh f2@PLT
+ jg f3@PLT
+*/
+
+void f1 (void);
+void f2 (void);
+void f3 (void);
+void f4 (void);
+
+#define TEST(T, U) \
+ void test_##U (T x, T y) \
+ { \
+ if (x == y) \
+ f1 (); \
+ else if (x < y) \
+ f2 (); \
+ else if (x > y) \
+ f3 (); \
+ else \
+ f4 (); \
+ }
+
+TEST (float, float)
+TEST (double, double)
+TEST (long double, longdouble)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-int-1.c b/gcc/testsuite/gcc.target/s390/spaceship-int-1.c
new file mode 100644
index 0000000..8ca2677
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-int-1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, -1\)} 4 optimized } } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 1\)} 5 optimized } } */
+/* { dg-final { scan-assembler-times {\tlhi} 9 } } */
+/* { dg-final { scan-assembler-times {\tloc} 18 } } */
+
+#define TEST(T, U) \
+ int test_##U (T x, T y) \
+ { \
+ if (x == y) \
+ return 0; \
+ else if (x < y) \
+ return -1; \
+ else \
+ return 1; \
+ }
+
+TEST(signed char, schar)
+TEST(unsigned char, uchar)
+TEST(char, char)
+
+TEST(short, sshort)
+TEST(unsigned short, ushort)
+
+TEST(int, sint)
+TEST(unsigned int, uint)
+
+TEST(long, slong)
+TEST(unsigned long, ulong)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-int-2.c b/gcc/testsuite/gcc.target/s390/spaceship-int-2.c
new file mode 100644
index 0000000..5f7975c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-int-2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, -1\)} 1 optimized } } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 1\)} 1 optimized } } */
+/* { dg-final { scan-assembler-times {\tvecg} 1 } } */
+/* { dg-final { scan-assembler-times {\tveclg} 1 } } */
+/* { dg-final { scan-assembler-times {\tvchlgs} 2 } } */
+/* { dg-final { scan-assembler-times {\tvceqgs} 2 } } */
+/* { dg-final { scan-assembler-times {\tlhi} 2 } } */
+/* { dg-final { scan-assembler-times {\tloc} 4 } } */
+
+#define TEST(T, U) \
+ int test_##U (T x, T y) \
+ { \
+ if (x == y) \
+ return 0; \
+ else if (x < y) \
+ return -1; \
+ else \
+ return 1; \
+ }
+
+TEST(__int128, sint128)
+TEST(unsigned __int128, uint128)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-int-3.c b/gcc/testsuite/gcc.target/s390/spaceship-int-3.c
new file mode 100644
index 0000000..46b0e4a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-int-3.c
@@ -0,0 +1,21 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O2 -march=z17 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, -1\)} 1 optimized } } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 1\)} 1 optimized } } */
+/* { dg-final { scan-assembler-times {\tvecq\t} 1 } } */
+/* { dg-final { scan-assembler-times {\tveclq\t} 1 } } */
+/* { dg-final { scan-assembler-times {\tloc} 4 } } */
+
+#define TEST(T, U) \
+ int test_##U (T x, T y) \
+ { \
+ if (x == y) \
+ return 0; \
+ else if (x < y) \
+ return -1; \
+ else \
+ return 1; \
+ }
+
+TEST(__int128, sint128)
+TEST(unsigned __int128, uint128)