; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NODOT
; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NODOT
; RUN: llc -mtriple=riscv32 -mattr=+v,+experimental-zvqdotq -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,DOT,DOT32
; RUN: llc -mtriple=riscv64 -mattr=+v,+experimental-zvqdotq -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,DOT,DOT64

define i32 @vqdot_vv(<16 x i8> %a, <16 x i8> %b) {
; NODOT-LABEL: vqdot_vv:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; NODOT-NEXT:    vsext.vf2 v12, v8
; NODOT-NEXT:    vsext.vf2 v14, v9
; NODOT-NEXT:    vwmul.vv v8, v12, v14
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.s.x v12, zero
; NODOT-NEXT:    vredsum.vs v8, v8, v12
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdot_vv:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v10, 0
; DOT-NEXT:    vqdot.vv v10, v8, v9
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vredsum.vs v8, v10, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.sext = sext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.sext
  %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul)
  ret i32 %res
}

define i32 @vqdot_vx_constant(<16 x i8> %a) {
; CHECK-LABEL: vqdot_vx_constant:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT:    vsext.vf2 v12, v8
; CHECK-NEXT:    li a0, 23
; CHECK-NEXT:    vwmul.vx v8, v12, a0
; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; CHECK-NEXT:    vmv.s.x v12, zero
; CHECK-NEXT:    vredsum.vs v8, v8, v12
; CHECK-NEXT:    vmv.x.s a0, v8
; CHECK-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %mul = mul <16 x i32> %a.sext, splat (i32 23)
  %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul)
  ret i32 %res
}

define i32 @vqdot_vx_constant_swapped(<16 x i8> %a) {
; CHECK-LABEL: vqdot_vx_constant_swapped:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT:    vsext.vf2 v12, v8
; CHECK-NEXT:    li a0, 23
; CHECK-NEXT:    vwmul.vx v8, v12, a0
; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; CHECK-NEXT:    vmv.s.x v12, zero
; CHECK-NEXT:    vredsum.vs v8, v8, v12
; CHECK-NEXT:    vmv.x.s a0, v8
; CHECK-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %mul = mul <16 x i32> splat (i32 23), %a.sext
  %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul)
  ret i32 %res
}

define i32 @vqdotu_vv(<16 x i8> %a, <16 x i8> %b) {
; NODOT-LABEL: vqdotu_vv:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
; NODOT-NEXT:    vwmulu.vv v10, v8, v9
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.s.x v8, zero
; NODOT-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
; NODOT-NEXT:    vwredsumu.vs v8, v10, v8
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotu_vv:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v10, 0
; DOT-NEXT:    vqdotu.vv v10, v8, v9
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vredsum.vs v8, v10, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.zext = zext <16 x i8> %a to <16 x i32>
  %b.zext = zext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.zext, %b.zext
  %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul)
  ret i32 %res
}

define i32 @vqdotu_vx_constant(<16 x i8> %a) {
; CHECK-LABEL: vqdotu_vx_constant:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT:    vzext.vf2 v12, v8
; CHECK-NEXT:    li a0, 123
; CHECK-NEXT:    vwmulu.vx v8, v12, a0
; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; CHECK-NEXT:    vmv.s.x v12, zero
; CHECK-NEXT:    vredsum.vs v8, v8, v12
; CHECK-NEXT:    vmv.x.s a0, v8
; CHECK-NEXT:    ret
entry:
  %a.zext = zext <16 x i8> %a to <16 x i32>
  %mul = mul <16 x i32> %a.zext, splat (i32 123)
  %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul)
  ret i32 %res
}

define i32 @vqdotsu_vv(<16 x i8> %a, <16 x i8> %b) {
; NODOT-LABEL: vqdotsu_vv:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; NODOT-NEXT:    vsext.vf2 v12, v8
; NODOT-NEXT:    vzext.vf2 v14, v9
; NODOT-NEXT:    vwmulsu.vv v8, v12, v14
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.s.x v12, zero
; NODOT-NEXT:    vredsum.vs v8, v8, v12
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotsu_vv:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v10, 0
; DOT-NEXT:    vqdotsu.vv v10, v8, v9
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vredsum.vs v8, v10, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.zext = zext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.zext
  %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul)
  ret i32 %res
}

define i32 @vqdotsu_vv_swapped(<16 x i8> %a, <16 x i8> %b) {
; NODOT-LABEL: vqdotsu_vv_swapped:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; NODOT-NEXT:    vsext.vf2 v12, v8
; NODOT-NEXT:    vzext.vf2 v14, v9
; NODOT-NEXT:    vwmulsu.vv v8, v12, v14
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.s.x v12, zero
; NODOT-NEXT:    vredsum.vs v8, v8, v12
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotsu_vv_swapped:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v10, 0
; DOT-NEXT:    vqdotsu.vv v10, v8, v9
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vredsum.vs v8, v10, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.zext = zext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %b.zext, %a.sext
  %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul)
  ret i32 %res
}

define i32 @vdotqsu_vx_constant(<16 x i8> %a) {
; CHECK-LABEL: vdotqsu_vx_constant:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT:    vsext.vf2 v12, v8
; CHECK-NEXT:    li a0, 123
; CHECK-NEXT:    vwmul.vx v8, v12, a0
; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; CHECK-NEXT:    vmv.s.x v12, zero
; CHECK-NEXT:    vredsum.vs v8, v8, v12
; CHECK-NEXT:    vmv.x.s a0, v8
; CHECK-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %mul = mul <16 x i32> %a.sext, splat (i32 123)
  %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul)
  ret i32 %res
}

define i32 @vdotqus_vx_constant(<16 x i8> %a) {
; CHECK-LABEL: vdotqus_vx_constant:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT:    vzext.vf2 v12, v8
; CHECK-NEXT:    li a0, -23
; CHECK-NEXT:    vmv.v.x v14, a0
; CHECK-NEXT:    vwmulsu.vv v8, v14, v12
; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; CHECK-NEXT:    vmv.s.x v12, zero
; CHECK-NEXT:    vredsum.vs v8, v8, v12
; CHECK-NEXT:    vmv.x.s a0, v8
; CHECK-NEXT:    ret
entry:
  %a.zext = zext <16 x i8> %a to <16 x i32>
  %mul = mul <16 x i32> %a.zext, splat (i32 -23)
  %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul)
  ret i32 %res
}

define i32 @reduce_of_sext(<16 x i8> %a) {
; NODOT-LABEL: reduce_of_sext:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; NODOT-NEXT:    vsext.vf4 v12, v8
; NODOT-NEXT:    vmv.s.x v8, zero
; NODOT-NEXT:    vredsum.vs v8, v12, v8
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: reduce_of_sext:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
; DOT-NEXT:    vmv.v.i v9, 1
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v10, 0
; DOT-NEXT:    vqdot.vv v10, v8, v9
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vredsum.vs v8, v10, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.ext = sext <16 x i8> %a to <16 x i32>
  %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %a.ext)
  ret i32 %res
}

define i32 @reduce_of_zext(<16 x i8> %a) {
; NODOT-LABEL: reduce_of_zext:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; NODOT-NEXT:    vzext.vf4 v12, v8
; NODOT-NEXT:    vmv.s.x v8, zero
; NODOT-NEXT:    vredsum.vs v8, v12, v8
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: reduce_of_zext:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
; DOT-NEXT:    vmv.v.i v9, 1
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v10, 0
; DOT-NEXT:    vqdotu.vv v10, v8, v9
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vredsum.vs v8, v10, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.ext = zext <16 x i8> %a to <16 x i32>
  %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %a.ext)
  ret i32 %res
}

define i32 @vqdot_vv_accum(<16 x i8> %a, <16 x i8> %b, <16 x i32> %x) {
; NODOT-LABEL: vqdot_vv_accum:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; NODOT-NEXT:    vsext.vf2 v10, v8
; NODOT-NEXT:    vsext.vf2 v16, v9
; NODOT-NEXT:    vwmacc.vv v12, v10, v16
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.s.x v8, zero
; NODOT-NEXT:    vredsum.vs v8, v12, v8
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdot_vv_accum:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv1r.v v16, v12
; DOT-NEXT:    vqdot.vv v16, v8, v9
; DOT-NEXT:    vsetivli zero, 4, e32, m4, tu, ma
; DOT-NEXT:    vmv.v.v v12, v16
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; DOT-NEXT:    vredsum.vs v8, v12, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.sext = sext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.sext
  %add = add <16 x i32> %mul, %x
  %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
  ret i32 %sum
}

define i32 @vqdotu_vv_accum(<16 x i8> %a, <16 x i8> %b, <16 x i32> %x) {
; NODOT-LABEL: vqdotu_vv_accum:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
; NODOT-NEXT:    vwmulu.vv v10, v8, v9
; NODOT-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
; NODOT-NEXT:    vwaddu.wv v12, v12, v10
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.s.x v8, zero
; NODOT-NEXT:    vredsum.vs v8, v12, v8
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotu_vv_accum:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv1r.v v16, v12
; DOT-NEXT:    vqdotu.vv v16, v8, v9
; DOT-NEXT:    vsetivli zero, 4, e32, m4, tu, ma
; DOT-NEXT:    vmv.v.v v12, v16
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; DOT-NEXT:    vredsum.vs v8, v12, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.zext = zext <16 x i8> %a to <16 x i32>
  %b.zext = zext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.zext, %b.zext
  %add = add <16 x i32> %mul, %x
  %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
  ret i32 %sum
}

define i32 @vqdotsu_vv_accum(<16 x i8> %a, <16 x i8> %b, <16 x i32> %x) {
; NODOT-LABEL: vqdotsu_vv_accum:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; NODOT-NEXT:    vsext.vf2 v10, v8
; NODOT-NEXT:    vzext.vf2 v16, v9
; NODOT-NEXT:    vwmaccsu.vv v12, v10, v16
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.s.x v8, zero
; NODOT-NEXT:    vredsum.vs v8, v12, v8
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotsu_vv_accum:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv1r.v v16, v12
; DOT-NEXT:    vqdotsu.vv v16, v8, v9
; DOT-NEXT:    vsetivli zero, 4, e32, m4, tu, ma
; DOT-NEXT:    vmv.v.v v12, v16
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; DOT-NEXT:    vredsum.vs v8, v12, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.zext = zext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.zext
  %add = add <16 x i32> %mul, %x
  %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
  ret i32 %sum
}

define i32 @vqdot_vv_scalar_add(<16 x i8> %a, <16 x i8> %b, i32 %x) {
; NODOT-LABEL: vqdot_vv_scalar_add:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; NODOT-NEXT:    vsext.vf2 v12, v8
; NODOT-NEXT:    vsext.vf2 v14, v9
; NODOT-NEXT:    vwmul.vv v8, v12, v14
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.s.x v12, a0
; NODOT-NEXT:    vredsum.vs v8, v8, v12
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdot_vv_scalar_add:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v10, 0
; DOT-NEXT:    vqdot.vv v10, v8, v9
; DOT-NEXT:    vmv.s.x v8, a0
; DOT-NEXT:    vredsum.vs v8, v10, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.sext = sext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.sext
  %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul)
  %add = add i32 %sum, %x
  ret i32 %add
}

define i32 @vqdotu_vv_scalar_add(<16 x i8> %a, <16 x i8> %b, i32 %x) {
; NODOT-LABEL: vqdotu_vv_scalar_add:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
; NODOT-NEXT:    vwmulu.vv v10, v8, v9
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.s.x v8, a0
; NODOT-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
; NODOT-NEXT:    vwredsumu.vs v8, v10, v8
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotu_vv_scalar_add:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v10, 0
; DOT-NEXT:    vqdotu.vv v10, v8, v9
; DOT-NEXT:    vmv.s.x v8, a0
; DOT-NEXT:    vredsum.vs v8, v10, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.zext = zext <16 x i8> %a to <16 x i32>
  %b.zext = zext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.zext, %b.zext
  %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul)
  %add = add i32 %sum, %x
  ret i32 %add
}

define i32 @vqdotsu_vv_scalar_add(<16 x i8> %a, <16 x i8> %b, i32 %x) {
; NODOT-LABEL: vqdotsu_vv_scalar_add:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; NODOT-NEXT:    vsext.vf2 v12, v8
; NODOT-NEXT:    vzext.vf2 v14, v9
; NODOT-NEXT:    vwmulsu.vv v8, v12, v14
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.s.x v12, a0
; NODOT-NEXT:    vredsum.vs v8, v8, v12
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotsu_vv_scalar_add:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v10, 0
; DOT-NEXT:    vqdotsu.vv v10, v8, v9
; DOT-NEXT:    vmv.s.x v8, a0
; DOT-NEXT:    vredsum.vs v8, v10, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.zext = zext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.zext
  %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul)
  %add = add i32 %sum, %x
  ret i32 %add
}

define i32 @vqdot_vv_split(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
; NODOT-LABEL: vqdot_vv_split:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; NODOT-NEXT:    vsext.vf2 v12, v8
; NODOT-NEXT:    vsext.vf2 v14, v9
; NODOT-NEXT:    vsext.vf2 v16, v10
; NODOT-NEXT:    vsext.vf2 v18, v11
; NODOT-NEXT:    vwmul.vv v8, v12, v14
; NODOT-NEXT:    vwmacc.vv v8, v16, v18
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.s.x v12, zero
; NODOT-NEXT:    vredsum.vs v8, v8, v12
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdot_vv_split:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v12, 0
; DOT-NEXT:    vqdot.vv v12, v8, v9
; DOT-NEXT:    vqdot.vv v12, v10, v11
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vredsum.vs v8, v12, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.sext = sext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.sext
  %c.sext = sext <16 x i8> %c to <16 x i32>
  %d.sext = sext <16 x i8> %d to <16 x i32>
  %mul2 = mul <16 x i32> %c.sext, %d.sext
  %add = add <16 x i32> %mul, %mul2
  %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
  ret i32 %sum
}

define <1 x i32> @vqdot_vv_partial_reduce_v1i32_v4i8(<4 x i8> %a, <4 x i8> %b) {
; NODOT-LABEL: vqdot_vv_partial_reduce_v1i32_v4i8:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
; NODOT-NEXT:    vsext.vf2 v10, v8
; NODOT-NEXT:    vsext.vf2 v8, v9
; NODOT-NEXT:    vwmul.vv v9, v10, v8
; NODOT-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v9, 3
; NODOT-NEXT:    vslidedown.vi v10, v9, 2
; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v8, v8, v9
; NODOT-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
; NODOT-NEXT:    vslidedown.vi v9, v9, 1
; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v9, v9, v10
; NODOT-NEXT:    vadd.vv v8, v9, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdot_vv_partial_reduce_v1i32_v4i8:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; DOT-NEXT:    vmv.s.x v10, zero
; DOT-NEXT:    vqdot.vv v10, v8, v9
; DOT-NEXT:    vmv1r.v v8, v10
; DOT-NEXT:    ret
entry:
  %a.sext = sext <4 x i8> %a to <4 x i32>
  %b.sext = sext <4 x i8> %b to <4 x i32>
  %mul = mul <4 x i32> %a.sext, %b.sext
  %res = call <1 x i32> @llvm.vector.partial.reduce.add(<1 x i32> zeroinitializer, <4 x i32> %mul)
  ret <1 x i32> %res
}

define <1 x i32> @vqdotu_vv_partial_reduce_v1i32_v4i8(<4 x i8> %a, <4 x i8> %b) {
; NODOT-LABEL: vqdotu_vv_partial_reduce_v1i32_v4i8:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
; NODOT-NEXT:    vwmulu.vv v10, v8, v9
; NODOT-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
; NODOT-NEXT:    vzext.vf2 v8, v10
; NODOT-NEXT:    vslidedown.vi v9, v8, 3
; NODOT-NEXT:    vslidedown.vi v10, v8, 2
; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v9, v9, v8
; NODOT-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v8, 1
; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v8, v8, v10
; NODOT-NEXT:    vadd.vv v8, v8, v9
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotu_vv_partial_reduce_v1i32_v4i8:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; DOT-NEXT:    vmv.s.x v10, zero
; DOT-NEXT:    vqdotu.vv v10, v8, v9
; DOT-NEXT:    vmv1r.v v8, v10
; DOT-NEXT:    ret
entry:
  %a.sext = zext <4 x i8> %a to <4 x i32>
  %b.sext = zext <4 x i8> %b to <4 x i32>
  %mul = mul <4 x i32> %a.sext, %b.sext
  %res = call <1 x i32> @llvm.vector.partial.reduce.add(<1 x i32> zeroinitializer, <4 x i32> %mul)
  ret <1 x i32> %res
}

define <1 x i32> @vqdotu_vx_partial_reduce(<4 x i8> %a, <4 x i8> %b) {
; NODOT-LABEL: vqdotu_vx_partial_reduce:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vzext.vf4 v9, v8
; NODOT-NEXT:    vsll.vi v8, v9, 7
; NODOT-NEXT:    vslidedown.vi v9, v8, 3
; NODOT-NEXT:    vslidedown.vi v10, v8, 2
; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v9, v9, v8
; NODOT-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v8, 1
; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v8, v8, v10
; NODOT-NEXT:    vadd.vv v8, v8, v9
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotu_vx_partial_reduce:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
; DOT-NEXT:    vmv.s.x v9, zero
; DOT-NEXT:    li a0, 128
; DOT-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
; DOT-NEXT:    vmv.v.x v10, a0
; DOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; DOT-NEXT:    vqdotu.vv v9, v8, v10
; DOT-NEXT:    vmv1r.v v8, v9
; DOT-NEXT:    ret
entry:
  %a.ext = zext <4 x i8> %a to <4 x i32>
  %mul = mul <4 x i32> %a.ext, splat (i32 128)
  %res = call <1 x i32> @llvm.vector.partial.reduce.add(<1 x i32> zeroinitializer, <4 x i32> %mul)
  ret <1 x i32> %res
}

define <1 x i32> @vqdot_vx_partial_reduce(<4 x i8> %a, <4 x i8> %b) {
; NODOT-LABEL: vqdot_vx_partial_reduce:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vsext.vf4 v9, v8
; NODOT-NEXT:    vsll.vi v8, v9, 7
; NODOT-NEXT:    vrsub.vi v8, v8, 0
; NODOT-NEXT:    vslidedown.vi v9, v8, 3
; NODOT-NEXT:    vslidedown.vi v10, v8, 2
; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v9, v9, v8
; NODOT-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v8, 1
; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v8, v8, v10
; NODOT-NEXT:    vadd.vv v8, v8, v9
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdot_vx_partial_reduce:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
; DOT-NEXT:    vmv.s.x v9, zero
; DOT-NEXT:    li a0, 128
; DOT-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
; DOT-NEXT:    vmv.v.x v10, a0
; DOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; DOT-NEXT:    vqdot.vv v9, v8, v10
; DOT-NEXT:    vmv1r.v v8, v9
; DOT-NEXT:    ret
entry:
  %a.ext = sext <4 x i8> %a to <4 x i32>
  %mul = mul <4 x i32> %a.ext, splat (i32 -128)
  %res = call <1 x i32> @llvm.vector.partial.reduce.add(<1 x i32> zeroinitializer, <4 x i32> %mul)
  ret <1 x i32> %res
}

define <1 x i32> @vqdotsu_vv_partial_reduce_v1i32_v4i8(<4 x i8> %a, <4 x i8> %b) {
; NODOT-LABEL: vqdotsu_vv_partial_reduce_v1i32_v4i8:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
; NODOT-NEXT:    vsext.vf2 v10, v8
; NODOT-NEXT:    vzext.vf2 v8, v9
; NODOT-NEXT:    vwmulsu.vv v9, v10, v8
; NODOT-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v9, 3
; NODOT-NEXT:    vslidedown.vi v10, v9, 2
; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v8, v8, v9
; NODOT-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
; NODOT-NEXT:    vslidedown.vi v9, v9, 1
; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v9, v9, v10
; NODOT-NEXT:    vadd.vv v8, v9, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotsu_vv_partial_reduce_v1i32_v4i8:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; DOT-NEXT:    vmv.s.x v10, zero
; DOT-NEXT:    vqdotsu.vv v10, v8, v9
; DOT-NEXT:    vmv1r.v v8, v10
; DOT-NEXT:    ret
entry:
  %a.sext = sext <4 x i8> %a to <4 x i32>
  %b.sext = zext <4 x i8> %b to <4 x i32>
  %mul = mul <4 x i32> %a.sext, %b.sext
  %res = call <1 x i32> @llvm.vector.partial.reduce.add(<1 x i32> zeroinitializer, <4 x i32> %mul)
  ret <1 x i32> %res
}

define <1 x i32> @vqdotsu_vv_partial_reduce_swapped(<4 x i8> %a, <4 x i8> %b) {
; NODOT-LABEL: vqdotsu_vv_partial_reduce_swapped:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
; NODOT-NEXT:    vsext.vf2 v10, v8
; NODOT-NEXT:    vzext.vf2 v8, v9
; NODOT-NEXT:    vwmulsu.vv v9, v10, v8
; NODOT-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v9, 3
; NODOT-NEXT:    vslidedown.vi v10, v9, 2
; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v8, v8, v9
; NODOT-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
; NODOT-NEXT:    vslidedown.vi v9, v9, 1
; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v9, v9, v10
; NODOT-NEXT:    vadd.vv v8, v9, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotsu_vv_partial_reduce_swapped:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; DOT-NEXT:    vmv.s.x v10, zero
; DOT-NEXT:    vqdotsu.vv v10, v8, v9
; DOT-NEXT:    vmv1r.v v8, v10
; DOT-NEXT:    ret
entry:
  %a.ext = sext <4 x i8> %a to <4 x i32>
  %b.ext = zext <4 x i8> %b to <4 x i32>
  %mul = mul <4 x i32> %b.ext, %a.ext
  %res = call <1 x i32> @llvm.vector.partial.reduce.add(<1 x i32> zeroinitializer, <4 x i32> %mul)
  ret <1 x i32> %res
}

define <1 x i32> @vqdotsu_vx_partial_reduce(<4 x i8> %a, <4 x i8> %b) {
; CHECK-LABEL: vqdotsu_vx_partial_reduce:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT:    vsext.vf4 v9, v8
; CHECK-NEXT:    vsll.vi v8, v9, 7
; CHECK-NEXT:    vslidedown.vi v9, v8, 3
; CHECK-NEXT:    vslidedown.vi v10, v8, 2
; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v9, v9, v8
; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
; CHECK-NEXT:    vslidedown.vi v8, v8, 1
; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v8, v8, v10
; CHECK-NEXT:    vadd.vv v8, v8, v9
; CHECK-NEXT:    ret
entry:
  %a.ext = sext <4 x i8> %a to <4 x i32>
  %mul = mul <4 x i32> %a.ext, splat (i32 128)
  %res = call <1 x i32> @llvm.vector.partial.reduce.add(<1 x i32> zeroinitializer, <4 x i32> %mul)
  ret <1 x i32> %res
}


define <2 x i32> @vqdot_vv_partial_reduce_v2i32_v8i8(<8 x i8> %a, <8 x i8> %b) {
; NODOT-LABEL: vqdot_vv_partial_reduce_v2i32_v8i8:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
; NODOT-NEXT:    vsext.vf2 v10, v8
; NODOT-NEXT:    vsext.vf2 v11, v9
; NODOT-NEXT:    vwmul.vv v8, v10, v11
; NODOT-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
; NODOT-NEXT:    vslidedown.vi v10, v8, 6
; NODOT-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v12, v10, v8
; NODOT-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
; NODOT-NEXT:    vslidedown.vi v10, v8, 4
; NODOT-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v8, 2
; NODOT-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; NODOT-NEXT:    vadd.vv v8, v8, v10
; NODOT-NEXT:    vadd.vv v8, v8, v12
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdot_vv_partial_reduce_v2i32_v8i8:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; DOT-NEXT:    vmv.v.i v10, 0
; DOT-NEXT:    vqdot.vv v10, v8, v9
; DOT-NEXT:    vmv1r.v v8, v10
; DOT-NEXT:    ret
entry:
  %a.sext = sext <8 x i8> %a to <8 x i32>
  %b.sext = sext <8 x i8> %b to <8 x i32>
  %mul = mul <8 x i32> %a.sext, %b.sext
  %res = call <2 x i32> @llvm.vector.partial.reduce.add(<2 x i32> zeroinitializer, <8 x i32> %mul)
  ret <2 x i32> %res
}

define <2 x i32> @vqdot_vv_partial_reduce_v2i32_v64i8(<64 x i8> %a, <64 x i8> %b) {
; CHECK-LABEL: vqdot_vv_partial_reduce_v2i32_v64i8:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    addi sp, sp, -16
; CHECK-NEXT:    .cfi_def_cfa_offset 16
; CHECK-NEXT:    csrr a0, vlenb
; CHECK-NEXT:    slli a1, a0, 2
; CHECK-NEXT:    add a0, a1, a0
; CHECK-NEXT:    sub sp, sp, a0
; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x05, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 5 * vlenb
; CHECK-NEXT:    li a0, 32
; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; CHECK-NEXT:    vsext.vf2 v24, v8
; CHECK-NEXT:    vsext.vf2 v28, v12
; CHECK-NEXT:    vwmul.vv v16, v24, v28
; CHECK-NEXT:    vsetivli zero, 2, e32, m8, ta, ma
; CHECK-NEXT:    vslidedown.vi v24, v16, 28
; CHECK-NEXT:    vslidedown.vi v0, v16, 26
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v24, v0, v24
; CHECK-NEXT:    csrr a1, vlenb
; CHECK-NEXT:    slli a1, a1, 2
; CHECK-NEXT:    add a1, sp, a1
; CHECK-NEXT:    addi a1, a1, 16
; CHECK-NEXT:    vs1r.v v24, (a1) # vscale x 8-byte Folded Spill
; CHECK-NEXT:    vsetivli zero, 2, e32, m8, ta, ma
; CHECK-NEXT:    vslidedown.vi v0, v16, 24
; CHECK-NEXT:    vslidedown.vi v24, v16, 22
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v24, v24, v0
; CHECK-NEXT:    csrr a1, vlenb
; CHECK-NEXT:    slli a2, a1, 1
; CHECK-NEXT:    add a1, a2, a1
; CHECK-NEXT:    add a1, sp, a1
; CHECK-NEXT:    addi a1, a1, 16
; CHECK-NEXT:    vs1r.v v24, (a1) # vscale x 8-byte Folded Spill
; CHECK-NEXT:    vsetivli zero, 2, e32, m8, ta, ma
; CHECK-NEXT:    vslidedown.vi v0, v16, 20
; CHECK-NEXT:    vslidedown.vi v24, v16, 18
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v24, v24, v0
; CHECK-NEXT:    csrr a1, vlenb
; CHECK-NEXT:    slli a1, a1, 1
; CHECK-NEXT:    add a1, sp, a1
; CHECK-NEXT:    addi a1, a1, 16
; CHECK-NEXT:    vs1r.v v24, (a1) # vscale x 8-byte Folded Spill
; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
; CHECK-NEXT:    vslidedown.vx v8, v8, a0
; CHECK-NEXT:    vslidedown.vx v12, v12, a0
; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; CHECK-NEXT:    vsext.vf2 v24, v8
; CHECK-NEXT:    vsext.vf2 v28, v12
; CHECK-NEXT:    vwmul.vv v8, v24, v28
; CHECK-NEXT:    vsetivli zero, 2, e32, m8, ta, ma
; CHECK-NEXT:    vslidedown.vi v0, v8, 28
; CHECK-NEXT:    vslidedown.vi v24, v8, 26
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v24, v24, v0
; CHECK-NEXT:    csrr a0, vlenb
; CHECK-NEXT:    add a0, sp, a0
; CHECK-NEXT:    addi a0, a0, 16
; CHECK-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
; CHECK-NEXT:    vsetivli zero, 2, e32, m8, ta, ma
; CHECK-NEXT:    vslidedown.vi v24, v8, 24
; CHECK-NEXT:    vslidedown.vi v0, v8, 22
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v24, v0, v24
; CHECK-NEXT:    addi a0, sp, 16
; CHECK-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
; CHECK-NEXT:    vsetivli zero, 2, e32, m8, ta, ma
; CHECK-NEXT:    vslidedown.vi v24, v8, 20
; CHECK-NEXT:    vslidedown.vi v0, v8, 18
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v7, v0, v24
; CHECK-NEXT:    vsetivli zero, 2, e32, m8, ta, ma
; CHECK-NEXT:    vslidedown.vi v24, v16, 16
; CHECK-NEXT:    vsetivli zero, 2, e32, m4, ta, ma
; CHECK-NEXT:    vslidedown.vi v28, v16, 14
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v6, v28, v24
; CHECK-NEXT:    vsetivli zero, 2, e32, m8, ta, ma
; CHECK-NEXT:    vslidedown.vi v24, v8, 16
; CHECK-NEXT:    vsetivli zero, 2, e32, m4, ta, ma
; CHECK-NEXT:    vslidedown.vi v28, v8, 14
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v5, v28, v24
; CHECK-NEXT:    vsetivli zero, 2, e32, m8, ta, ma
; CHECK-NEXT:    vslidedown.vi v24, v16, 30
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v4, v24, v16
; CHECK-NEXT:    vsetivli zero, 2, e32, m4, ta, ma
; CHECK-NEXT:    vslidedown.vi v20, v16, 12
; CHECK-NEXT:    vslidedown.vi v24, v16, 10
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v3, v24, v20
; CHECK-NEXT:    vsetivli zero, 2, e32, m8, ta, ma
; CHECK-NEXT:    vslidedown.vi v24, v8, 30
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v24, v24, v8
; CHECK-NEXT:    vsetivli zero, 2, e32, m4, ta, ma
; CHECK-NEXT:    vslidedown.vi v12, v8, 12
; CHECK-NEXT:    vslidedown.vi v20, v8, 10
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v25, v20, v12
; CHECK-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
; CHECK-NEXT:    vslidedown.vi v20, v16, 6
; CHECK-NEXT:    vslidedown.vi v22, v16, 4
; CHECK-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
; CHECK-NEXT:    vslidedown.vi v21, v16, 2
; CHECK-NEXT:    vsetivli zero, 2, e32, m4, ta, ma
; CHECK-NEXT:    vslidedown.vi v12, v16, 8
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v16, v20, v12
; CHECK-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
; CHECK-NEXT:    vslidedown.vi v12, v8, 6
; CHECK-NEXT:    vslidedown.vi v14, v8, 4
; CHECK-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
; CHECK-NEXT:    vslidedown.vi v13, v8, 2
; CHECK-NEXT:    vsetivli zero, 2, e32, m4, ta, ma
; CHECK-NEXT:    vslidedown.vi v8, v8, 8
; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v8, v12, v8
; CHECK-NEXT:    vadd.vv v9, v21, v22
; CHECK-NEXT:    vadd.vv v10, v13, v14
; CHECK-NEXT:    csrr a0, vlenb
; CHECK-NEXT:    slli a0, a0, 2
; CHECK-NEXT:    add a0, sp, a0
; CHECK-NEXT:    addi a0, a0, 16
; CHECK-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
; CHECK-NEXT:    vadd.vv v11, v11, v4
; CHECK-NEXT:    csrr a0, vlenb
; CHECK-NEXT:    slli a1, a0, 1
; CHECK-NEXT:    add a0, a1, a0
; CHECK-NEXT:    add a0, sp, a0
; CHECK-NEXT:    addi a0, a0, 16
; CHECK-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
; CHECK-NEXT:    csrr a0, vlenb
; CHECK-NEXT:    slli a0, a0, 1
; CHECK-NEXT:    add a0, sp, a0
; CHECK-NEXT:    addi a0, a0, 16
; CHECK-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
; CHECK-NEXT:    vadd.vv v12, v13, v12
; CHECK-NEXT:    vadd.vv v13, v3, v6
; CHECK-NEXT:    vadd.vv v9, v9, v16
; CHECK-NEXT:    vadd.vv v11, v12, v11
; CHECK-NEXT:    vadd.vv v9, v9, v13
; CHECK-NEXT:    addi a0, sp, 16
; CHECK-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
; CHECK-NEXT:    vadd.vv v12, v7, v12
; CHECK-NEXT:    vadd.vv v13, v25, v5
; CHECK-NEXT:    vadd.vv v8, v10, v8
; CHECK-NEXT:    vadd.vv v9, v9, v11
; CHECK-NEXT:    vadd.vv v9, v24, v9
; CHECK-NEXT:    csrr a0, vlenb
; CHECK-NEXT:    add a0, sp, a0
; CHECK-NEXT:    addi a0, a0, 16
; CHECK-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
; CHECK-NEXT:    vadd.vv v9, v10, v9
; CHECK-NEXT:    vadd.vv v9, v12, v9
; CHECK-NEXT:    vadd.vv v8, v8, v13
; CHECK-NEXT:    vadd.vv v8, v8, v9
; CHECK-NEXT:    csrr a0, vlenb
; CHECK-NEXT:    slli a1, a0, 2
; CHECK-NEXT:    add a0, a1, a0
; CHECK-NEXT:    add sp, sp, a0
; CHECK-NEXT:    .cfi_def_cfa sp, 16
; CHECK-NEXT:    addi sp, sp, 16
; CHECK-NEXT:    .cfi_def_cfa_offset 0
; CHECK-NEXT:    ret
entry:
  %a.sext = sext <64 x i8> %a to <64 x i32>
  %b.sext = sext <64 x i8> %b to <64 x i32>
  %mul = mul <64 x i32> %a.sext, %b.sext
  %res = call <2 x i32> @llvm.vector.partial.reduce.add(<2 x i32> zeroinitializer, <64 x i32> %mul)
  ret <2 x i32> %res
}

define <4 x i32> @vqdot_vv_partial_reduce_v4i32_v16i8(<16 x i8> %a, <16 x i8> %b) {
; NODOT-LABEL: vqdot_vv_partial_reduce_v4i32_v16i8:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; NODOT-NEXT:    vsext.vf2 v12, v8
; NODOT-NEXT:    vsext.vf2 v14, v9
; NODOT-NEXT:    vwmul.vv v8, v12, v14
; NODOT-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
; NODOT-NEXT:    vslidedown.vi v12, v8, 12
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vadd.vv v16, v12, v8
; NODOT-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
; NODOT-NEXT:    vslidedown.vi v12, v8, 8
; NODOT-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v8, 4
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vadd.vv v8, v8, v12
; NODOT-NEXT:    vadd.vv v8, v8, v16
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdot_vv_partial_reduce_v4i32_v16i8:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v10, 0
; DOT-NEXT:    vqdot.vv v10, v8, v9
; DOT-NEXT:    vmv.v.v v8, v10
; DOT-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.sext = sext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.sext
  %res = call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> zeroinitializer, <16 x i32> %mul)
  ret <4 x i32> %res
}

define <16 x i32> @vqdot_vv_partial_reduce_v16i32_v64i8(<64 x i8> %a, <64 x i8> %b) {
; NODOT-LABEL: vqdot_vv_partial_reduce_v16i32_v64i8:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    li a0, 32
; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; NODOT-NEXT:    vsext.vf2 v16, v8
; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
; NODOT-NEXT:    vslidedown.vx v8, v8, a0
; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; NODOT-NEXT:    vsext.vf2 v20, v12
; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
; NODOT-NEXT:    vslidedown.vx v12, v12, a0
; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; NODOT-NEXT:    vsext.vf2 v24, v8
; NODOT-NEXT:    vsext.vf2 v28, v12
; NODOT-NEXT:    vwmul.vv v8, v16, v20
; NODOT-NEXT:    vwmul.vv v16, v24, v28
; NODOT-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
; NODOT-NEXT:    vslidedown.vi v24, v8, 16
; NODOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; NODOT-NEXT:    vadd.vv v8, v24, v8
; NODOT-NEXT:    vadd.vv v24, v8, v16
; NODOT-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v16, 16
; NODOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; NODOT-NEXT:    vadd.vv v8, v8, v24
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdot_vv_partial_reduce_v16i32_v64i8:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; DOT-NEXT:    vmv.v.i v16, 0
; DOT-NEXT:    vqdot.vv v16, v8, v12
; DOT-NEXT:    vmv.v.v v8, v16
; DOT-NEXT:    ret
entry:
  %a.sext = sext <64 x i8> %a to <64 x i32>
  %b.sext = sext <64 x i8> %b to <64 x i32>
  %mul = mul <64 x i32> %a.sext, %b.sext
  %res = call <16 x i32> @llvm.vector.partial.reduce.add(<16 x i32> zeroinitializer, <64 x i32> %mul)
  ret <16 x i32> %res
}

define <4 x i32> @vqdot_vv_partial_reduce_m1_accum(<16 x i8> %a, <16 x i8> %b, <4 x i32> %accum) {
; NODOT-LABEL: vqdot_vv_partial_reduce_m1_accum:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; NODOT-NEXT:    vsext.vf2 v16, v8
; NODOT-NEXT:    vsext.vf2 v18, v9
; NODOT-NEXT:    vwmul.vv v12, v16, v18
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vadd.vv v16, v10, v12
; NODOT-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v12, 12
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vadd.vv v16, v8, v16
; NODOT-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v12, 8
; NODOT-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
; NODOT-NEXT:    vslidedown.vi v10, v12, 4
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vadd.vv v8, v10, v8
; NODOT-NEXT:    vadd.vv v8, v8, v16
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdot_vv_partial_reduce_m1_accum:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vqdot.vv v10, v8, v9
; DOT-NEXT:    vmv.v.v v8, v10
; DOT-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.sext = sext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.sext
  %res = call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> %accum, <16 x i32> %mul)
  ret <4 x i32> %res
}

define <16 x i32> @vqdot_vv_partial_reduce3(<16 x i8> %a, <16 x i8> %b) {
; CHECK-LABEL: vqdot_vv_partial_reduce3:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT:    vsext.vf2 v12, v8
; CHECK-NEXT:    vsext.vf2 v14, v9
; CHECK-NEXT:    vwmul.vv v8, v12, v14
; CHECK-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.sext = sext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.sext
  %res = call <16 x i32> @llvm.vector.partial.reduce.add.nvx8i32.nvx16i32.nvx16i32(<16 x i32> %mul, <16 x i32> zeroinitializer)
  ret <16 x i32> %res
}

; Test legalization - type split
define <64 x i32> @vqdotsu_vv_partial_v64i32_v256i8(<256 x i8> %a, <256 x i8> %b) {
; NODOT-LABEL: vqdotsu_vv_partial_v64i32_v256i8:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    addi sp, sp, -16
; NODOT-NEXT:    .cfi_def_cfa_offset 16
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 3
; NODOT-NEXT:    mv a2, a1
; NODOT-NEXT:    slli a1, a1, 2
; NODOT-NEXT:    add a1, a1, a2
; NODOT-NEXT:    sub sp, sp, a1
; NODOT-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x28, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 40 * vlenb
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 4
; NODOT-NEXT:    add a1, sp, a1
; NODOT-NEXT:    addi a1, a1, 16
; NODOT-NEXT:    vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 5
; NODOT-NEXT:    add a1, sp, a1
; NODOT-NEXT:    addi a1, a1, 16
; NODOT-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
; NODOT-NEXT:    addi a1, a0, 128
; NODOT-NEXT:    li a2, 128
; NODOT-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
; NODOT-NEXT:    vle8.v v0, (a0)
; NODOT-NEXT:    csrr a0, vlenb
; NODOT-NEXT:    slli a0, a0, 3
; NODOT-NEXT:    mv a3, a0
; NODOT-NEXT:    slli a0, a0, 1
; NODOT-NEXT:    add a0, a0, a3
; NODOT-NEXT:    add a0, sp, a0
; NODOT-NEXT:    addi a0, a0, 16
; NODOT-NEXT:    vs8r.v v0, (a0) # vscale x 64-byte Folded Spill
; NODOT-NEXT:    li a0, 32
; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
; NODOT-NEXT:    vslidedown.vx v24, v8, a0
; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; NODOT-NEXT:    vsext.vf2 v8, v24
; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
; NODOT-NEXT:    vslidedown.vx v12, v0, a0
; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; NODOT-NEXT:    vzext.vf2 v4, v12
; NODOT-NEXT:    vwmulsu.vv v24, v8, v4
; NODOT-NEXT:    csrr a3, vlenb
; NODOT-NEXT:    slli a3, a3, 5
; NODOT-NEXT:    add a3, sp, a3
; NODOT-NEXT:    addi a3, a3, 16
; NODOT-NEXT:    vl8r.v v8, (a3) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vsext.vf2 v4, v8
; NODOT-NEXT:    csrr a3, vlenb
; NODOT-NEXT:    slli a3, a3, 3
; NODOT-NEXT:    mv a4, a3
; NODOT-NEXT:    slli a3, a3, 1
; NODOT-NEXT:    add a3, a3, a4
; NODOT-NEXT:    add a3, sp, a3
; NODOT-NEXT:    addi a3, a3, 16
; NODOT-NEXT:    vl8r.v v8, (a3) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vzext.vf2 v0, v8
; NODOT-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
; NODOT-NEXT:    vle8.v v8, (a1)
; NODOT-NEXT:    addi a1, sp, 16
; NODOT-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; NODOT-NEXT:    vwmaccsu.vv v24, v4, v0
; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
; NODOT-NEXT:    vslidedown.vx v4, v16, a0
; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; NODOT-NEXT:    vsext.vf2 v12, v4
; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
; NODOT-NEXT:    vslidedown.vx v4, v8, a0
; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; NODOT-NEXT:    vzext.vf2 v16, v4
; NODOT-NEXT:    vwmulsu.vv v0, v12, v16
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 4
; NODOT-NEXT:    add a1, sp, a1
; NODOT-NEXT:    addi a1, a1, 16
; NODOT-NEXT:    vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vsext.vf2 v12, v16
; NODOT-NEXT:    vzext.vf2 v20, v8
; NODOT-NEXT:    vwmaccsu.vv v0, v12, v20
; NODOT-NEXT:    li a1, 64
; NODOT-NEXT:    csrr a2, vlenb
; NODOT-NEXT:    slli a2, a2, 5
; NODOT-NEXT:    add a2, sp, a2
; NODOT-NEXT:    addi a2, a2, 16
; NODOT-NEXT:    vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
; NODOT-NEXT:    vslidedown.vx v8, v16, a1
; NODOT-NEXT:    csrr a2, vlenb
; NODOT-NEXT:    slli a2, a2, 5
; NODOT-NEXT:    add a2, sp, a2
; NODOT-NEXT:    addi a2, a2, 16
; NODOT-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
; NODOT-NEXT:    csrr a2, vlenb
; NODOT-NEXT:    slli a2, a2, 3
; NODOT-NEXT:    mv a3, a2
; NODOT-NEXT:    slli a2, a2, 1
; NODOT-NEXT:    add a2, a2, a3
; NODOT-NEXT:    add a2, sp, a2
; NODOT-NEXT:    addi a2, a2, 16
; NODOT-NEXT:    vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vslidedown.vx v8, v16, a1
; NODOT-NEXT:    csrr a2, vlenb
; NODOT-NEXT:    slli a2, a2, 3
; NODOT-NEXT:    add a2, sp, a2
; NODOT-NEXT:    addi a2, a2, 16
; NODOT-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
; NODOT-NEXT:    csrr a2, vlenb
; NODOT-NEXT:    slli a2, a2, 5
; NODOT-NEXT:    add a2, sp, a2
; NODOT-NEXT:    addi a2, a2, 16
; NODOT-NEXT:    vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; NODOT-NEXT:    vsext.vf2 v16, v8
; NODOT-NEXT:    csrr a2, vlenb
; NODOT-NEXT:    slli a2, a2, 3
; NODOT-NEXT:    add a2, sp, a2
; NODOT-NEXT:    addi a2, a2, 16
; NODOT-NEXT:    vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vzext.vf2 v20, v8
; NODOT-NEXT:    vwmaccsu.vv v24, v16, v20
; NODOT-NEXT:    csrr a2, vlenb
; NODOT-NEXT:    slli a2, a2, 4
; NODOT-NEXT:    add a2, sp, a2
; NODOT-NEXT:    addi a2, a2, 16
; NODOT-NEXT:    vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
; NODOT-NEXT:    vslidedown.vx v16, v16, a1
; NODOT-NEXT:    addi a2, sp, 16
; NODOT-NEXT:    vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vslidedown.vx v8, v8, a1
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 3
; NODOT-NEXT:    mv a2, a1
; NODOT-NEXT:    slli a1, a1, 1
; NODOT-NEXT:    add a1, a1, a2
; NODOT-NEXT:    add a1, sp, a1
; NODOT-NEXT:    addi a1, a1, 16
; NODOT-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; NODOT-NEXT:    vsext.vf2 v8, v16
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 4
; NODOT-NEXT:    add a1, sp, a1
; NODOT-NEXT:    addi a1, a1, 16
; NODOT-NEXT:    vs4r.v v8, (a1) # vscale x 32-byte Folded Spill
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 3
; NODOT-NEXT:    mv a2, a1
; NODOT-NEXT:    slli a1, a1, 1
; NODOT-NEXT:    add a1, a1, a2
; NODOT-NEXT:    add a1, sp, a1
; NODOT-NEXT:    addi a1, a1, 16
; NODOT-NEXT:    vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vzext.vf2 v20, v8
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 4
; NODOT-NEXT:    add a1, sp, a1
; NODOT-NEXT:    addi a1, a1, 16
; NODOT-NEXT:    vl4r.v v8, (a1) # vscale x 32-byte Folded Reload
; NODOT-NEXT:    vwmaccsu.vv v0, v8, v20
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 5
; NODOT-NEXT:    add a1, sp, a1
; NODOT-NEXT:    addi a1, a1, 16
; NODOT-NEXT:    vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
; NODOT-NEXT:    vslidedown.vx v20, v8, a0
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 3
; NODOT-NEXT:    add a1, sp, a1
; NODOT-NEXT:    addi a1, a1, 16
; NODOT-NEXT:    vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vslidedown.vx v8, v8, a0
; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; NODOT-NEXT:    vsext.vf2 v12, v20
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 5
; NODOT-NEXT:    add a1, sp, a1
; NODOT-NEXT:    addi a1, a1, 16
; NODOT-NEXT:    vs4r.v v12, (a1) # vscale x 32-byte Folded Spill
; NODOT-NEXT:    vzext.vf2 v12, v8
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 5
; NODOT-NEXT:    add a1, sp, a1
; NODOT-NEXT:    addi a1, a1, 16
; NODOT-NEXT:    vl4r.v v8, (a1) # vscale x 32-byte Folded Reload
; NODOT-NEXT:    vwmaccsu.vv v24, v8, v12
; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
; NODOT-NEXT:    vslidedown.vx v12, v16, a0
; NODOT-NEXT:    csrr a1, vlenb
; NODOT-NEXT:    slli a1, a1, 3
; NODOT-NEXT:    mv a2, a1
; NODOT-NEXT:    slli a1, a1, 1
; NODOT-NEXT:    add a1, a1, a2
; NODOT-NEXT:    add a1, sp, a1
; NODOT-NEXT:    addi a1, a1, 16
; NODOT-NEXT:    vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
; NODOT-NEXT:    vslidedown.vx v8, v16, a0
; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
; NODOT-NEXT:    vsext.vf2 v16, v12
; NODOT-NEXT:    vzext.vf2 v12, v8
; NODOT-NEXT:    vwmaccsu.vv v0, v16, v12
; NODOT-NEXT:    vmv8r.v v8, v24
; NODOT-NEXT:    vmv8r.v v16, v0
; NODOT-NEXT:    csrr a0, vlenb
; NODOT-NEXT:    slli a0, a0, 3
; NODOT-NEXT:    mv a1, a0
; NODOT-NEXT:    slli a0, a0, 2
; NODOT-NEXT:    add a0, a0, a1
; NODOT-NEXT:    add sp, sp, a0
; NODOT-NEXT:    .cfi_def_cfa sp, 16
; NODOT-NEXT:    addi sp, sp, 16
; NODOT-NEXT:    .cfi_def_cfa_offset 0
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotsu_vv_partial_v64i32_v256i8:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    addi sp, sp, -16
; DOT-NEXT:    .cfi_def_cfa_offset 16
; DOT-NEXT:    csrr a1, vlenb
; DOT-NEXT:    slli a1, a1, 5
; DOT-NEXT:    sub sp, sp, a1
; DOT-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
; DOT-NEXT:    csrr a1, vlenb
; DOT-NEXT:    slli a1, a1, 3
; DOT-NEXT:    mv a2, a1
; DOT-NEXT:    slli a1, a1, 1
; DOT-NEXT:    add a1, a1, a2
; DOT-NEXT:    add a1, sp, a1
; DOT-NEXT:    addi a1, a1, 16
; DOT-NEXT:    vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
; DOT-NEXT:    csrr a1, vlenb
; DOT-NEXT:    slli a1, a1, 4
; DOT-NEXT:    add a1, sp, a1
; DOT-NEXT:    addi a1, a1, 16
; DOT-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
; DOT-NEXT:    addi a1, a0, 128
; DOT-NEXT:    li a2, 128
; DOT-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
; DOT-NEXT:    vle8.v v8, (a0)
; DOT-NEXT:    csrr a0, vlenb
; DOT-NEXT:    slli a0, a0, 3
; DOT-NEXT:    add a0, sp, a0
; DOT-NEXT:    addi a0, a0, 16
; DOT-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
; DOT-NEXT:    li a0, 32
; DOT-NEXT:    vle8.v v8, (a1)
; DOT-NEXT:    addi a1, sp, 16
; DOT-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
; DOT-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
; DOT-NEXT:    vmv.v.i v24, 0
; DOT-NEXT:    vmv.v.i v0, 0
; DOT-NEXT:    csrr a0, vlenb
; DOT-NEXT:    slli a0, a0, 4
; DOT-NEXT:    add a0, sp, a0
; DOT-NEXT:    addi a0, a0, 16
; DOT-NEXT:    vl8r.v v16, (a0) # vscale x 64-byte Folded Reload
; DOT-NEXT:    csrr a0, vlenb
; DOT-NEXT:    slli a0, a0, 3
; DOT-NEXT:    add a0, sp, a0
; DOT-NEXT:    addi a0, a0, 16
; DOT-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
; DOT-NEXT:    vqdotsu.vv v0, v16, v8
; DOT-NEXT:    csrr a0, vlenb
; DOT-NEXT:    slli a0, a0, 3
; DOT-NEXT:    mv a1, a0
; DOT-NEXT:    slli a0, a0, 1
; DOT-NEXT:    add a0, a0, a1
; DOT-NEXT:    add a0, sp, a0
; DOT-NEXT:    addi a0, a0, 16
; DOT-NEXT:    vl8r.v v16, (a0) # vscale x 64-byte Folded Reload
; DOT-NEXT:    addi a0, sp, 16
; DOT-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
; DOT-NEXT:    vqdotsu.vv v24, v16, v8
; DOT-NEXT:    vmv.v.v v8, v0
; DOT-NEXT:    vmv.v.v v16, v24
; DOT-NEXT:    csrr a0, vlenb
; DOT-NEXT:    slli a0, a0, 5
; DOT-NEXT:    add sp, sp, a0
; DOT-NEXT:    .cfi_def_cfa sp, 16
; DOT-NEXT:    addi sp, sp, 16
; DOT-NEXT:    .cfi_def_cfa_offset 0
; DOT-NEXT:    ret
entry:
  %a.ext = sext <256 x i8> %a to <256 x i32>
  %b.ext = zext <256 x i8> %b to <256 x i32>
  %mul = mul <256 x i32> %b.ext, %a.ext
  %res = call <64 x i32> @llvm.vector.partial.reduce.add(<64 x i32> zeroinitializer, <256 x i32> %mul)
  ret <64 x i32> %res
}

; Test legalization - integer promote
define <4 x i31> @vqdotsu_vv_partial_v4i31_v16i7(<16 x i7> %a, <16 x i7> %b) {
; NODOT-LABEL: vqdotsu_vv_partial_v4i31_v16i7:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; NODOT-NEXT:    vzext.vf4 v12, v8
; NODOT-NEXT:    li a0, 127
; NODOT-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
; NODOT-NEXT:    vand.vx v16, v9, a0
; NODOT-NEXT:    lui a0, 524288
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vsll.vi v8, v12, 25
; NODOT-NEXT:    addi a0, a0, -1
; NODOT-NEXT:    vsra.vi v8, v8, 25
; NODOT-NEXT:    vzext.vf4 v12, v16
; NODOT-NEXT:    vmul.vv v8, v12, v8
; NODOT-NEXT:    vand.vx v8, v8, a0
; NODOT-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
; NODOT-NEXT:    vslidedown.vi v12, v8, 12
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vadd.vv v16, v12, v8
; NODOT-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
; NODOT-NEXT:    vslidedown.vi v12, v8, 8
; NODOT-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v8, 4
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vadd.vv v8, v8, v12
; NODOT-NEXT:    vadd.vv v8, v8, v16
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotsu_vv_partial_v4i31_v16i7:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    li a0, 127
; DOT-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
; DOT-NEXT:    vadd.vv v8, v8, v8
; DOT-NEXT:    vand.vx v9, v9, a0
; DOT-NEXT:    vsra.vi v10, v8, 1
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v8, 0
; DOT-NEXT:    vqdotsu.vv v8, v10, v9
; DOT-NEXT:    ret
entry:
  %a.ext = sext <16 x i7> %a to <16 x i31>
  %b.ext = zext <16 x i7> %b to <16 x i31>
  %mul = mul <16 x i31> %b.ext, %a.ext
  %res = call <4 x i31> @llvm.vector.partial.reduce.add(<4 x i31> zeroinitializer, <16 x i31> %mul)
  ret <4 x i31> %res
}


; Test legalization - expand
define <1 x i32> @vqdotsu_vv_partial_v1i32_v2i8(<2 x i8> %a, <2 x i8> %b) {
; CHECK-LABEL: vqdotsu_vv_partial_v1i32_v2i8:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
; CHECK-NEXT:    vsext.vf2 v10, v8
; CHECK-NEXT:    vzext.vf2 v8, v9
; CHECK-NEXT:    vwmulsu.vv v9, v10, v8
; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
; CHECK-NEXT:    vslidedown.vi v8, v9, 1
; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v8, v8, v9
; CHECK-NEXT:    ret
entry:
  %a.ext = sext <2 x i8> %a to <2 x i32>
  %b.ext = zext <2 x i8> %b to <2 x i32>
  %mul = mul <2 x i32> %b.ext, %a.ext
  %res = call <1 x i32> @llvm.vector.partial.reduce.add(<1 x i32> zeroinitializer, <2 x i32> %mul)
  ret <1 x i32> %res
}

; TODO: This isn't legal, but we could split it into two halves, and use
; a pair of slides + two vqdotsu_vv here.  Or alternatively, the mul
; sequence + one vredsum, or a vadd reduce tree.
define <1 x i32> @vqdotsu_vv_partial_v1i32_v8i8(<8 x i8> %a, <8 x i8> %b) {
; CHECK-LABEL: vqdotsu_vv_partial_v1i32_v8i8:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT:    vsext.vf2 v10, v8
; CHECK-NEXT:    vzext.vf2 v11, v9
; CHECK-NEXT:    vwmulsu.vv v8, v10, v11
; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
; CHECK-NEXT:    vslidedown.vi v10, v8, 6
; CHECK-NEXT:    vslidedown.vi v12, v8, 5
; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v12, v12, v10
; CHECK-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
; CHECK-NEXT:    vslidedown.vi v10, v8, 7
; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
; CHECK-NEXT:    vslidedown.vi v11, v8, 3
; CHECK-NEXT:    vslidedown.vi v13, v8, 2
; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v10, v10, v8
; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
; CHECK-NEXT:    vslidedown.vi v14, v8, 1
; CHECK-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
; CHECK-NEXT:    vslidedown.vi v8, v8, 4
; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
; CHECK-NEXT:    vadd.vv v8, v11, v8
; CHECK-NEXT:    vadd.vv v9, v14, v13
; CHECK-NEXT:    vadd.vv v10, v12, v10
; CHECK-NEXT:    vadd.vv v8, v9, v8
; CHECK-NEXT:    vadd.vv v8, v8, v10
; CHECK-NEXT:    ret
entry:
  %a.ext = sext <8 x i8> %a to <8 x i32>
  %b.ext = zext <8 x i8> %b to <8 x i32>
  %mul = mul <8 x i32> %b.ext, %a.ext
  %res = call <1 x i32> @llvm.vector.partial.reduce.add(<1 x i32> zeroinitializer, <8 x i32> %mul)
  ret <1 x i32> %res
}


define <4 x i32> @partial_of_sext(<16 x i8> %a) {
; NODOT-LABEL: partial_of_sext:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; NODOT-NEXT:    vsext.vf4 v12, v8
; NODOT-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v12, 12
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vadd.vv v16, v8, v12
; NODOT-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v12, 8
; NODOT-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
; NODOT-NEXT:    vslidedown.vi v10, v12, 4
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vadd.vv v8, v10, v8
; NODOT-NEXT:    vadd.vv v8, v8, v16
; NODOT-NEXT:    ret
;
; DOT-LABEL: partial_of_sext:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
; DOT-NEXT:    vmv.v.i v10, 1
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v9, 0
; DOT-NEXT:    vqdot.vv v9, v8, v10
; DOT-NEXT:    vmv.v.v v8, v9
; DOT-NEXT:    ret
entry:
  %a.ext = sext <16 x i8> %a to <16 x i32>
  %res = call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> zeroinitializer, <16 x i32> %a.ext)
  ret <4 x i32> %res
}

define <4 x i32> @partial_of_zext(<16 x i8> %a) {
; NODOT-LABEL: partial_of_zext:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; NODOT-NEXT:    vzext.vf4 v12, v8
; NODOT-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v12, 12
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vadd.vv v16, v8, v12
; NODOT-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
; NODOT-NEXT:    vslidedown.vi v8, v12, 8
; NODOT-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
; NODOT-NEXT:    vslidedown.vi v10, v12, 4
; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; NODOT-NEXT:    vadd.vv v8, v10, v8
; NODOT-NEXT:    vadd.vv v8, v8, v16
; NODOT-NEXT:    ret
;
; DOT-LABEL: partial_of_zext:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
; DOT-NEXT:    vmv.v.i v10, 1
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv.v.i v9, 0
; DOT-NEXT:    vqdotu.vv v9, v8, v10
; DOT-NEXT:    vmv.v.v v8, v9
; DOT-NEXT:    ret
entry:
  %a.ext = zext <16 x i8> %a to <16 x i32>
  %res = call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> zeroinitializer, <16 x i32> %a.ext)
  ret <4 x i32> %res
}

define i32 @vqdot_vv_accum_disjoint_or(<16 x i8> %a, <16 x i8> %b, <16 x i32> %x) {
; NODOT-LABEL: vqdot_vv_accum_disjoint_or:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; NODOT-NEXT:    vsext.vf2 v16, v8
; NODOT-NEXT:    vsext.vf2 v18, v9
; NODOT-NEXT:    vwmul.vv v8, v16, v18
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vor.vv v8, v8, v12
; NODOT-NEXT:    vmv.s.x v12, zero
; NODOT-NEXT:    vredsum.vs v8, v8, v12
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdot_vv_accum_disjoint_or:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv1r.v v16, v12
; DOT-NEXT:    vqdot.vv v16, v8, v9
; DOT-NEXT:    vsetivli zero, 4, e32, m4, tu, ma
; DOT-NEXT:    vmv.v.v v12, v16
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; DOT-NEXT:    vredsum.vs v8, v12, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.sext = sext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.sext
  %add = or disjoint <16 x i32> %mul, %x
  %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
  ret i32 %sum
}

define i32 @vqdot_vv_accum_or(<16 x i8> %a, <16 x i8> %b, <16 x i32> %x) {
; CHECK-LABEL: vqdot_vv_accum_or:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT:    vsext.vf2 v16, v8
; CHECK-NEXT:    vsext.vf2 v18, v9
; CHECK-NEXT:    vwmul.vv v8, v16, v18
; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; CHECK-NEXT:    vor.vv v8, v8, v12
; CHECK-NEXT:    vmv.s.x v12, zero
; CHECK-NEXT:    vredsum.vs v8, v8, v12
; CHECK-NEXT:    vmv.x.s a0, v8
; CHECK-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.sext = sext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.sext
  %add = or <16 x i32> %mul, %x
  %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
  ret i32 %sum
}

define i32 @vqdotu_vv_accum_disjoint_or(<16 x i8> %a, <16 x i8> %b, <16 x i32> %x) {
; NODOT-LABEL: vqdotu_vv_accum_disjoint_or:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
; NODOT-NEXT:    vwmulu.vv v10, v8, v9
; NODOT-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
; NODOT-NEXT:    vwaddu.wv v12, v12, v10
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vmv.s.x v8, zero
; NODOT-NEXT:    vredsum.vs v8, v12, v8
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotu_vv_accum_disjoint_or:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv1r.v v16, v12
; DOT-NEXT:    vqdotu.vv v16, v8, v9
; DOT-NEXT:    vsetivli zero, 4, e32, m4, tu, ma
; DOT-NEXT:    vmv.v.v v12, v16
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; DOT-NEXT:    vredsum.vs v8, v12, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.zext = zext <16 x i8> %a to <16 x i32>
  %b.zext = zext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.zext, %b.zext
  %add = or disjoint <16 x i32> %mul, %x
  %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
  ret i32 %sum
}

define i32 @vqdotsu_vv_accum_disjoint_or(<16 x i8> %a, <16 x i8> %b, <16 x i32> %x) {
; NODOT-LABEL: vqdotsu_vv_accum_disjoint_or:
; NODOT:       # %bb.0: # %entry
; NODOT-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
; NODOT-NEXT:    vsext.vf2 v16, v8
; NODOT-NEXT:    vzext.vf2 v18, v9
; NODOT-NEXT:    vwmulsu.vv v8, v16, v18
; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
; NODOT-NEXT:    vor.vv v8, v8, v12
; NODOT-NEXT:    vmv.s.x v12, zero
; NODOT-NEXT:    vredsum.vs v8, v8, v12
; NODOT-NEXT:    vmv.x.s a0, v8
; NODOT-NEXT:    ret
;
; DOT-LABEL: vqdotsu_vv_accum_disjoint_or:
; DOT:       # %bb.0: # %entry
; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT:    vmv1r.v v16, v12
; DOT-NEXT:    vqdotsu.vv v16, v8, v9
; DOT-NEXT:    vsetivli zero, 4, e32, m4, tu, ma
; DOT-NEXT:    vmv.v.v v12, v16
; DOT-NEXT:    vmv.s.x v8, zero
; DOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
; DOT-NEXT:    vredsum.vs v8, v12, v8
; DOT-NEXT:    vmv.x.s a0, v8
; DOT-NEXT:    ret
entry:
  %a.sext = sext <16 x i8> %a to <16 x i32>
  %b.zext = zext <16 x i8> %b to <16 x i32>
  %mul = mul <16 x i32> %a.sext, %b.zext
  %add = or disjoint <16 x i32> %mul, %x
  %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
  ret i32 %sum
}

;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; DOT32: {{.*}}
; DOT64: {{.*}}