; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64 -mattr=+sve2 %s -o - | FileCheck %s

define <16 x i1> @whilewr_8(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.b, x0, x1
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 1)
  ret <16 x i1> %0
}

define <8 x i1> @whilewr_16(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.h, x0, x1
; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.8b, v0.8h
; CHECK-NEXT:    ret
entry:
  %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr %a, ptr %b, i64 2)
  ret <8 x i1> %0
}

define <4 x i1> @whilewr_32(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.s, x0, x1
; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret
entry:
  %0 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr %a, ptr %b, i64 4)
  ret <4 x i1> %0
}

define <2 x i1> @whilewr_64(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_64:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.d, x0, x1
; CHECK-NEXT:    mov z0.d, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.2s, v0.2d
; CHECK-NEXT:    ret
entry:
  %0 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1(ptr %a, ptr %b, i64 8)
  ret <2 x i1> %0
}

define <16 x i1> @whilerw_8(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilerw p0.b, x0, x1
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.raw.mask.v16i1(ptr %a, ptr %b, i64 1)
  ret <16 x i1> %0
}

define <8 x i1> @whilerw_16(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilerw p0.h, x0, x1
; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.8b, v0.8h
; CHECK-NEXT:    ret
entry:
  %0 = call <8 x i1> @llvm.loop.dependence.raw.mask.v8i1(ptr %a, ptr %b, i64 2)
  ret <8 x i1> %0
}

define <4 x i1> @whilerw_32(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilerw p0.s, x0, x1
; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret
entry:
  %0 = call <4 x i1> @llvm.loop.dependence.raw.mask.v4i1(ptr %a, ptr %b, i64 4)
  ret <4 x i1> %0
}

define <2 x i1> @whilerw_64(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_64:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilerw p0.d, x0, x1
; CHECK-NEXT:    mov z0.d, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.2s, v0.2d
; CHECK-NEXT:    ret
entry:
  %0 = call <2 x i1> @llvm.loop.dependence.raw.mask.v2i1(ptr %a, ptr %b, i64 8)
  ret <2 x i1> %0
}

define <32 x i1> @whilewr_8_split(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_8_split:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    add x9, x0, #16
; CHECK-NEXT:    whilewr p0.b, x0, x1
; CHECK-NEXT:    whilewr p1.b, x9, x1
; CHECK-NEXT:    adrp x9, .LCPI8_0
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI8_0]
; CHECK-NEXT:    mov z1.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    shl v0.16b, v0.16b, #7
; CHECK-NEXT:    shl v1.16b, v1.16b, #7
; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
; CHECK-NEXT:    zip1 v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    zip1 v1.16b, v1.16b, v3.16b
; CHECK-NEXT:    addv h0, v0.8h
; CHECK-NEXT:    addv h1, v1.8h
; CHECK-NEXT:    str h0, [x8]
; CHECK-NEXT:    str h1, [x8, #2]
; CHECK-NEXT:    ret
entry:
  %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1(ptr %a, ptr %b, i64 1)
  ret <32 x i1> %0
}

define <64 x i1> @whilewr_8_split2(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_8_split2:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    add x9, x0, #48
; CHECK-NEXT:    whilewr p0.b, x0, x1
; CHECK-NEXT:    add x10, x0, #16
; CHECK-NEXT:    whilewr p1.b, x9, x1
; CHECK-NEXT:    add x9, x0, #32
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    whilewr p0.b, x9, x1
; CHECK-NEXT:    adrp x9, .LCPI9_0
; CHECK-NEXT:    mov z1.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    whilewr p1.b, x10, x1
; CHECK-NEXT:    ldr q4, [x9, :lo12:.LCPI9_0]
; CHECK-NEXT:    mov z2.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    mov z3.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    shl v0.16b, v0.16b, #7
; CHECK-NEXT:    shl v1.16b, v1.16b, #7
; CHECK-NEXT:    shl v2.16b, v2.16b, #7
; CHECK-NEXT:    shl v3.16b, v3.16b, #7
; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
; CHECK-NEXT:    cmlt v2.16b, v2.16b, #0
; CHECK-NEXT:    cmlt v3.16b, v3.16b, #0
; CHECK-NEXT:    and v0.16b, v0.16b, v4.16b
; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
; CHECK-NEXT:    and v2.16b, v2.16b, v4.16b
; CHECK-NEXT:    and v3.16b, v3.16b, v4.16b
; CHECK-NEXT:    ext v4.16b, v0.16b, v0.16b, #8
; CHECK-NEXT:    ext v5.16b, v1.16b, v1.16b, #8
; CHECK-NEXT:    ext v6.16b, v2.16b, v2.16b, #8
; CHECK-NEXT:    ext v7.16b, v3.16b, v3.16b, #8
; CHECK-NEXT:    zip1 v0.16b, v0.16b, v4.16b
; CHECK-NEXT:    zip1 v1.16b, v1.16b, v5.16b
; CHECK-NEXT:    zip1 v2.16b, v2.16b, v6.16b
; CHECK-NEXT:    zip1 v3.16b, v3.16b, v7.16b
; CHECK-NEXT:    addv h0, v0.8h
; CHECK-NEXT:    addv h1, v1.8h
; CHECK-NEXT:    addv h2, v2.8h
; CHECK-NEXT:    addv h3, v3.8h
; CHECK-NEXT:    str h0, [x8]
; CHECK-NEXT:    str h1, [x8, #6]
; CHECK-NEXT:    str h2, [x8, #4]
; CHECK-NEXT:    str h3, [x8, #2]
; CHECK-NEXT:    ret
entry:
  %0 = call <64 x i1> @llvm.loop.dependence.war.mask.v64i1(ptr %a, ptr %b, i64 1)
  ret <64 x i1> %0
}

define <16 x i1> @whilewr_16_expand(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_16_expand:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    index z0.d, #0, #1
; CHECK-NEXT:    sub x8, x1, x0
; CHECK-NEXT:    add x8, x8, x8, lsr #63
; CHECK-NEXT:    asr x8, x8, #1
; CHECK-NEXT:    mov z1.d, z0.d
; CHECK-NEXT:    mov z2.d, z0.d
; CHECK-NEXT:    mov z4.d, z0.d
; CHECK-NEXT:    mov z5.d, z0.d
; CHECK-NEXT:    mov z6.d, z0.d
; CHECK-NEXT:    mov z7.d, z0.d
; CHECK-NEXT:    mov z16.d, z0.d
; CHECK-NEXT:    dup v3.2d, x8
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    add z1.d, z1.d, #12 // =0xc
; CHECK-NEXT:    add z2.d, z2.d, #10 // =0xa
; CHECK-NEXT:    add z4.d, z4.d, #8 // =0x8
; CHECK-NEXT:    add z5.d, z5.d, #6 // =0x6
; CHECK-NEXT:    add z6.d, z6.d, #4 // =0x4
; CHECK-NEXT:    add z7.d, z7.d, #2 // =0x2
; CHECK-NEXT:    add z16.d, z16.d, #14 // =0xe
; CHECK-NEXT:    cmhi v0.2d, v3.2d, v0.2d
; CHECK-NEXT:    cset w8, lt
; CHECK-NEXT:    cmhi v1.2d, v3.2d, v1.2d
; CHECK-NEXT:    cmhi v2.2d, v3.2d, v2.2d
; CHECK-NEXT:    cmhi v4.2d, v3.2d, v4.2d
; CHECK-NEXT:    cmhi v5.2d, v3.2d, v5.2d
; CHECK-NEXT:    cmhi v6.2d, v3.2d, v6.2d
; CHECK-NEXT:    cmhi v16.2d, v3.2d, v16.2d
; CHECK-NEXT:    cmhi v3.2d, v3.2d, v7.2d
; CHECK-NEXT:    uzp1 v2.4s, v4.4s, v2.4s
; CHECK-NEXT:    uzp1 v4.4s, v6.4s, v5.4s
; CHECK-NEXT:    uzp1 v1.4s, v1.4s, v16.4s
; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v3.4s
; CHECK-NEXT:    uzp1 v1.8h, v2.8h, v1.8h
; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    dup v1.16b, w8
; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 2)
  ret <16 x i1> %0
}

define <32 x i1> @whilewr_16_expand2(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_16_expand2:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    sub x9, x1, x0
; CHECK-NEXT:    index z0.d, #0, #1
; CHECK-NEXT:    sub x10, x9, #32
; CHECK-NEXT:    add x9, x9, x9, lsr #63
; CHECK-NEXT:    add x10, x10, x10, lsr #63
; CHECK-NEXT:    asr x9, x9, #1
; CHECK-NEXT:    asr x10, x10, #1
; CHECK-NEXT:    mov z1.d, z0.d
; CHECK-NEXT:    mov z2.d, z0.d
; CHECK-NEXT:    mov z3.d, z0.d
; CHECK-NEXT:    mov z4.d, z0.d
; CHECK-NEXT:    mov z5.d, z0.d
; CHECK-NEXT:    mov z6.d, z0.d
; CHECK-NEXT:    dup v7.2d, x9
; CHECK-NEXT:    dup v16.2d, x10
; CHECK-NEXT:    add z1.d, z1.d, #12 // =0xc
; CHECK-NEXT:    add z2.d, z2.d, #10 // =0xa
; CHECK-NEXT:    cmp x10, #1
; CHECK-NEXT:    add z3.d, z3.d, #8 // =0x8
; CHECK-NEXT:    add z4.d, z4.d, #6 // =0x6
; CHECK-NEXT:    add z5.d, z5.d, #4 // =0x4
; CHECK-NEXT:    add z6.d, z6.d, #2 // =0x2
; CHECK-NEXT:    cmhi v17.2d, v7.2d, v0.2d
; CHECK-NEXT:    cmhi v18.2d, v16.2d, v0.2d
; CHECK-NEXT:    add z0.d, z0.d, #14 // =0xe
; CHECK-NEXT:    cmhi v19.2d, v7.2d, v1.2d
; CHECK-NEXT:    cmhi v20.2d, v7.2d, v2.2d
; CHECK-NEXT:    cmhi v21.2d, v7.2d, v3.2d
; CHECK-NEXT:    cmhi v22.2d, v7.2d, v4.2d
; CHECK-NEXT:    cmhi v23.2d, v7.2d, v5.2d
; CHECK-NEXT:    cmhi v24.2d, v7.2d, v6.2d
; CHECK-NEXT:    cmhi v1.2d, v16.2d, v1.2d
; CHECK-NEXT:    cmhi v2.2d, v16.2d, v2.2d
; CHECK-NEXT:    cmhi v3.2d, v16.2d, v3.2d
; CHECK-NEXT:    cmhi v4.2d, v16.2d, v4.2d
; CHECK-NEXT:    cmhi v7.2d, v7.2d, v0.2d
; CHECK-NEXT:    cmhi v5.2d, v16.2d, v5.2d
; CHECK-NEXT:    cmhi v6.2d, v16.2d, v6.2d
; CHECK-NEXT:    cset w10, lt
; CHECK-NEXT:    cmhi v0.2d, v16.2d, v0.2d
; CHECK-NEXT:    uzp1 v16.4s, v21.4s, v20.4s
; CHECK-NEXT:    cmp x9, #1
; CHECK-NEXT:    uzp1 v20.4s, v23.4s, v22.4s
; CHECK-NEXT:    uzp1 v17.4s, v17.4s, v24.4s
; CHECK-NEXT:    cset w9, lt
; CHECK-NEXT:    uzp1 v2.4s, v3.4s, v2.4s
; CHECK-NEXT:    uzp1 v3.4s, v19.4s, v7.4s
; CHECK-NEXT:    uzp1 v4.4s, v5.4s, v4.4s
; CHECK-NEXT:    uzp1 v5.4s, v18.4s, v6.4s
; CHECK-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
; CHECK-NEXT:    uzp1 v1.8h, v17.8h, v20.8h
; CHECK-NEXT:    uzp1 v3.8h, v16.8h, v3.8h
; CHECK-NEXT:    uzp1 v4.8h, v5.8h, v4.8h
; CHECK-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
; CHECK-NEXT:    dup v2.16b, w9
; CHECK-NEXT:    adrp x9, .LCPI11_0
; CHECK-NEXT:    uzp1 v1.16b, v1.16b, v3.16b
; CHECK-NEXT:    dup v3.16b, w10
; CHECK-NEXT:    uzp1 v0.16b, v4.16b, v0.16b
; CHECK-NEXT:    orr v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI11_0]
; CHECK-NEXT:    orr v0.16b, v0.16b, v3.16b
; CHECK-NEXT:    shl v1.16b, v1.16b, #7
; CHECK-NEXT:    shl v0.16b, v0.16b, #7
; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    ext v2.16b, v1.16b, v1.16b, #8
; CHECK-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
; CHECK-NEXT:    zip1 v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    zip1 v0.16b, v0.16b, v3.16b
; CHECK-NEXT:    addv h1, v1.8h
; CHECK-NEXT:    addv h0, v0.8h
; CHECK-NEXT:    str h1, [x8]
; CHECK-NEXT:    str h0, [x8, #2]
; CHECK-NEXT:    ret
entry:
  %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1(ptr %a, ptr %b, i64 2)
  ret <32 x i1> %0
}

define <8 x i1> @whilewr_32_expand(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_32_expand:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    index z0.d, #0, #1
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    add x9, x8, #3
; CHECK-NEXT:    csel x8, x9, x8, mi
; CHECK-NEXT:    asr x8, x8, #2
; CHECK-NEXT:    mov z2.d, z0.d
; CHECK-NEXT:    mov z3.d, z0.d
; CHECK-NEXT:    mov z4.d, z0.d
; CHECK-NEXT:    dup v1.2d, x8
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    cset w8, lt
; CHECK-NEXT:    add z4.d, z4.d, #6 // =0x6
; CHECK-NEXT:    add z2.d, z2.d, #4 // =0x4
; CHECK-NEXT:    add z3.d, z3.d, #2 // =0x2
; CHECK-NEXT:    cmhi v0.2d, v1.2d, v0.2d
; CHECK-NEXT:    cmhi v4.2d, v1.2d, v4.2d
; CHECK-NEXT:    cmhi v2.2d, v1.2d, v2.2d
; CHECK-NEXT:    cmhi v1.2d, v1.2d, v3.2d
; CHECK-NEXT:    uzp1 v2.4s, v2.4s, v4.4s
; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
; CHECK-NEXT:    dup v1.8b, w8
; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
; CHECK-NEXT:    xtn v0.8b, v0.8h
; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-NEXT:    ret
entry:
  %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr %a, ptr %b, i64 4)
  ret <8 x i1> %0
}

define <16 x i1> @whilewr_32_expand2(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_32_expand2:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    index z0.d, #0, #1
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    add x9, x8, #3
; CHECK-NEXT:    csel x8, x9, x8, mi
; CHECK-NEXT:    asr x8, x8, #2
; CHECK-NEXT:    mov z1.d, z0.d
; CHECK-NEXT:    mov z2.d, z0.d
; CHECK-NEXT:    mov z4.d, z0.d
; CHECK-NEXT:    mov z5.d, z0.d
; CHECK-NEXT:    mov z6.d, z0.d
; CHECK-NEXT:    mov z7.d, z0.d
; CHECK-NEXT:    mov z16.d, z0.d
; CHECK-NEXT:    dup v3.2d, x8
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    add z1.d, z1.d, #12 // =0xc
; CHECK-NEXT:    add z2.d, z2.d, #10 // =0xa
; CHECK-NEXT:    add z4.d, z4.d, #8 // =0x8
; CHECK-NEXT:    add z5.d, z5.d, #6 // =0x6
; CHECK-NEXT:    add z6.d, z6.d, #4 // =0x4
; CHECK-NEXT:    add z7.d, z7.d, #2 // =0x2
; CHECK-NEXT:    add z16.d, z16.d, #14 // =0xe
; CHECK-NEXT:    cmhi v0.2d, v3.2d, v0.2d
; CHECK-NEXT:    cset w8, lt
; CHECK-NEXT:    cmhi v1.2d, v3.2d, v1.2d
; CHECK-NEXT:    cmhi v2.2d, v3.2d, v2.2d
; CHECK-NEXT:    cmhi v4.2d, v3.2d, v4.2d
; CHECK-NEXT:    cmhi v5.2d, v3.2d, v5.2d
; CHECK-NEXT:    cmhi v6.2d, v3.2d, v6.2d
; CHECK-NEXT:    cmhi v16.2d, v3.2d, v16.2d
; CHECK-NEXT:    cmhi v3.2d, v3.2d, v7.2d
; CHECK-NEXT:    uzp1 v2.4s, v4.4s, v2.4s
; CHECK-NEXT:    uzp1 v4.4s, v6.4s, v5.4s
; CHECK-NEXT:    uzp1 v1.4s, v1.4s, v16.4s
; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v3.4s
; CHECK-NEXT:    uzp1 v1.8h, v2.8h, v1.8h
; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    dup v1.16b, w8
; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 4)
  ret <16 x i1> %0
}

define <32 x i1> @whilewr_32_expand3(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_32_expand3:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x9, x1, x0
; CHECK-NEXT:    index z0.d, #0, #1
; CHECK-NEXT:    add x10, x9, #3
; CHECK-NEXT:    sub x11, x9, #61
; CHECK-NEXT:    csel x10, x10, x9, mi
; CHECK-NEXT:    subs x9, x9, #64
; CHECK-NEXT:    csel x9, x11, x9, mi
; CHECK-NEXT:    asr x10, x10, #2
; CHECK-NEXT:    asr x9, x9, #2
; CHECK-NEXT:    mov z1.d, z0.d
; CHECK-NEXT:    mov z2.d, z0.d
; CHECK-NEXT:    mov z3.d, z0.d
; CHECK-NEXT:    mov z4.d, z0.d
; CHECK-NEXT:    mov z5.d, z0.d
; CHECK-NEXT:    mov z6.d, z0.d
; CHECK-NEXT:    dup v7.2d, x10
; CHECK-NEXT:    dup v16.2d, x9
; CHECK-NEXT:    add z1.d, z1.d, #12 // =0xc
; CHECK-NEXT:    add z2.d, z2.d, #10 // =0xa
; CHECK-NEXT:    cmp x9, #1
; CHECK-NEXT:    add z3.d, z3.d, #8 // =0x8
; CHECK-NEXT:    add z4.d, z4.d, #6 // =0x6
; CHECK-NEXT:    add z5.d, z5.d, #4 // =0x4
; CHECK-NEXT:    add z6.d, z6.d, #2 // =0x2
; CHECK-NEXT:    cmhi v17.2d, v7.2d, v0.2d
; CHECK-NEXT:    cmhi v18.2d, v16.2d, v0.2d
; CHECK-NEXT:    add z0.d, z0.d, #14 // =0xe
; CHECK-NEXT:    cmhi v19.2d, v7.2d, v1.2d
; CHECK-NEXT:    cmhi v20.2d, v7.2d, v2.2d
; CHECK-NEXT:    cmhi v21.2d, v7.2d, v3.2d
; CHECK-NEXT:    cmhi v22.2d, v7.2d, v4.2d
; CHECK-NEXT:    cmhi v23.2d, v7.2d, v5.2d
; CHECK-NEXT:    cmhi v24.2d, v7.2d, v6.2d
; CHECK-NEXT:    cmhi v1.2d, v16.2d, v1.2d
; CHECK-NEXT:    cmhi v2.2d, v16.2d, v2.2d
; CHECK-NEXT:    cmhi v3.2d, v16.2d, v3.2d
; CHECK-NEXT:    cmhi v4.2d, v16.2d, v4.2d
; CHECK-NEXT:    cmhi v7.2d, v7.2d, v0.2d
; CHECK-NEXT:    cmhi v5.2d, v16.2d, v5.2d
; CHECK-NEXT:    cmhi v6.2d, v16.2d, v6.2d
; CHECK-NEXT:    cset w9, lt
; CHECK-NEXT:    cmhi v0.2d, v16.2d, v0.2d
; CHECK-NEXT:    uzp1 v16.4s, v21.4s, v20.4s
; CHECK-NEXT:    cmp x10, #1
; CHECK-NEXT:    uzp1 v20.4s, v23.4s, v22.4s
; CHECK-NEXT:    uzp1 v17.4s, v17.4s, v24.4s
; CHECK-NEXT:    cset w10, lt
; CHECK-NEXT:    uzp1 v2.4s, v3.4s, v2.4s
; CHECK-NEXT:    uzp1 v3.4s, v19.4s, v7.4s
; CHECK-NEXT:    uzp1 v4.4s, v5.4s, v4.4s
; CHECK-NEXT:    uzp1 v5.4s, v18.4s, v6.4s
; CHECK-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
; CHECK-NEXT:    uzp1 v1.8h, v17.8h, v20.8h
; CHECK-NEXT:    uzp1 v3.8h, v16.8h, v3.8h
; CHECK-NEXT:    uzp1 v4.8h, v5.8h, v4.8h
; CHECK-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
; CHECK-NEXT:    dup v2.16b, w10
; CHECK-NEXT:    uzp1 v1.16b, v1.16b, v3.16b
; CHECK-NEXT:    dup v3.16b, w9
; CHECK-NEXT:    adrp x9, .LCPI14_0
; CHECK-NEXT:    uzp1 v0.16b, v4.16b, v0.16b
; CHECK-NEXT:    orr v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI14_0]
; CHECK-NEXT:    orr v0.16b, v0.16b, v3.16b
; CHECK-NEXT:    shl v1.16b, v1.16b, #7
; CHECK-NEXT:    shl v0.16b, v0.16b, #7
; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    ext v2.16b, v1.16b, v1.16b, #8
; CHECK-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
; CHECK-NEXT:    zip1 v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    zip1 v0.16b, v0.16b, v3.16b
; CHECK-NEXT:    addv h1, v1.8h
; CHECK-NEXT:    addv h0, v0.8h
; CHECK-NEXT:    str h1, [x8]
; CHECK-NEXT:    str h0, [x8, #2]
; CHECK-NEXT:    ret
entry:
  %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1(ptr %a, ptr %b, i64 4)
  ret <32 x i1> %0
}

define <4 x i1> @whilewr_64_expand(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_64_expand:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    index z0.d, #0, #1
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    add x9, x8, #7
; CHECK-NEXT:    csel x8, x9, x8, mi
; CHECK-NEXT:    asr x8, x8, #3
; CHECK-NEXT:    mov z1.d, z0.d
; CHECK-NEXT:    dup v2.2d, x8
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    cset w8, lt
; CHECK-NEXT:    add z1.d, z1.d, #2 // =0x2
; CHECK-NEXT:    cmhi v0.2d, v2.2d, v0.2d
; CHECK-NEXT:    cmhi v1.2d, v2.2d, v1.2d
; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
; CHECK-NEXT:    dup v1.4h, w8
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-NEXT:    ret
entry:
  %0 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr %a, ptr %b, i64 8)
  ret <4 x i1> %0
}

define <8 x i1> @whilewr_64_expand2(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_64_expand2:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    index z0.d, #0, #1
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    add x9, x8, #7
; CHECK-NEXT:    csel x8, x9, x8, mi
; CHECK-NEXT:    asr x8, x8, #3
; CHECK-NEXT:    mov z2.d, z0.d
; CHECK-NEXT:    mov z3.d, z0.d
; CHECK-NEXT:    mov z4.d, z0.d
; CHECK-NEXT:    dup v1.2d, x8
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    cset w8, lt
; CHECK-NEXT:    add z4.d, z4.d, #6 // =0x6
; CHECK-NEXT:    add z2.d, z2.d, #4 // =0x4
; CHECK-NEXT:    add z3.d, z3.d, #2 // =0x2
; CHECK-NEXT:    cmhi v0.2d, v1.2d, v0.2d
; CHECK-NEXT:    cmhi v4.2d, v1.2d, v4.2d
; CHECK-NEXT:    cmhi v2.2d, v1.2d, v2.2d
; CHECK-NEXT:    cmhi v1.2d, v1.2d, v3.2d
; CHECK-NEXT:    uzp1 v2.4s, v2.4s, v4.4s
; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
; CHECK-NEXT:    dup v1.8b, w8
; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
; CHECK-NEXT:    xtn v0.8b, v0.8h
; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-NEXT:    ret
entry:
  %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr %a, ptr %b, i64 8)
  ret <8 x i1> %0
}

define <16 x i1> @whilewr_64_expand3(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_64_expand3:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    index z0.d, #0, #1
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    add x9, x8, #7
; CHECK-NEXT:    csel x8, x9, x8, mi
; CHECK-NEXT:    asr x8, x8, #3
; CHECK-NEXT:    mov z1.d, z0.d
; CHECK-NEXT:    mov z2.d, z0.d
; CHECK-NEXT:    mov z4.d, z0.d
; CHECK-NEXT:    mov z5.d, z0.d
; CHECK-NEXT:    mov z6.d, z0.d
; CHECK-NEXT:    mov z7.d, z0.d
; CHECK-NEXT:    mov z16.d, z0.d
; CHECK-NEXT:    dup v3.2d, x8
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    add z1.d, z1.d, #12 // =0xc
; CHECK-NEXT:    add z2.d, z2.d, #10 // =0xa
; CHECK-NEXT:    add z4.d, z4.d, #8 // =0x8
; CHECK-NEXT:    add z5.d, z5.d, #6 // =0x6
; CHECK-NEXT:    add z6.d, z6.d, #4 // =0x4
; CHECK-NEXT:    add z7.d, z7.d, #2 // =0x2
; CHECK-NEXT:    add z16.d, z16.d, #14 // =0xe
; CHECK-NEXT:    cmhi v0.2d, v3.2d, v0.2d
; CHECK-NEXT:    cset w8, lt
; CHECK-NEXT:    cmhi v1.2d, v3.2d, v1.2d
; CHECK-NEXT:    cmhi v2.2d, v3.2d, v2.2d
; CHECK-NEXT:    cmhi v4.2d, v3.2d, v4.2d
; CHECK-NEXT:    cmhi v5.2d, v3.2d, v5.2d
; CHECK-NEXT:    cmhi v6.2d, v3.2d, v6.2d
; CHECK-NEXT:    cmhi v16.2d, v3.2d, v16.2d
; CHECK-NEXT:    cmhi v3.2d, v3.2d, v7.2d
; CHECK-NEXT:    uzp1 v2.4s, v4.4s, v2.4s
; CHECK-NEXT:    uzp1 v4.4s, v6.4s, v5.4s
; CHECK-NEXT:    uzp1 v1.4s, v1.4s, v16.4s
; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v3.4s
; CHECK-NEXT:    uzp1 v1.8h, v2.8h, v1.8h
; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    dup v1.16b, w8
; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 8)
  ret <16 x i1> %0
}

define <32 x i1> @whilewr_64_expand4(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_64_expand4:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x9, x1, x0
; CHECK-NEXT:    index z0.d, #0, #1
; CHECK-NEXT:    add x10, x9, #7
; CHECK-NEXT:    sub x11, x9, #121
; CHECK-NEXT:    csel x10, x10, x9, mi
; CHECK-NEXT:    subs x9, x9, #128
; CHECK-NEXT:    csel x9, x11, x9, mi
; CHECK-NEXT:    asr x10, x10, #3
; CHECK-NEXT:    asr x9, x9, #3
; CHECK-NEXT:    mov z1.d, z0.d
; CHECK-NEXT:    mov z2.d, z0.d
; CHECK-NEXT:    mov z3.d, z0.d
; CHECK-NEXT:    mov z4.d, z0.d
; CHECK-NEXT:    mov z5.d, z0.d
; CHECK-NEXT:    mov z6.d, z0.d
; CHECK-NEXT:    dup v7.2d, x10
; CHECK-NEXT:    dup v16.2d, x9
; CHECK-NEXT:    add z1.d, z1.d, #12 // =0xc
; CHECK-NEXT:    add z2.d, z2.d, #10 // =0xa
; CHECK-NEXT:    cmp x9, #1
; CHECK-NEXT:    add z3.d, z3.d, #8 // =0x8
; CHECK-NEXT:    add z4.d, z4.d, #6 // =0x6
; CHECK-NEXT:    add z5.d, z5.d, #4 // =0x4
; CHECK-NEXT:    add z6.d, z6.d, #2 // =0x2
; CHECK-NEXT:    cmhi v17.2d, v7.2d, v0.2d
; CHECK-NEXT:    cmhi v18.2d, v16.2d, v0.2d
; CHECK-NEXT:    add z0.d, z0.d, #14 // =0xe
; CHECK-NEXT:    cmhi v19.2d, v7.2d, v1.2d
; CHECK-NEXT:    cmhi v20.2d, v7.2d, v2.2d
; CHECK-NEXT:    cmhi v21.2d, v7.2d, v3.2d
; CHECK-NEXT:    cmhi v22.2d, v7.2d, v4.2d
; CHECK-NEXT:    cmhi v23.2d, v7.2d, v5.2d
; CHECK-NEXT:    cmhi v24.2d, v7.2d, v6.2d
; CHECK-NEXT:    cmhi v1.2d, v16.2d, v1.2d
; CHECK-NEXT:    cmhi v2.2d, v16.2d, v2.2d
; CHECK-NEXT:    cmhi v3.2d, v16.2d, v3.2d
; CHECK-NEXT:    cmhi v4.2d, v16.2d, v4.2d
; CHECK-NEXT:    cmhi v7.2d, v7.2d, v0.2d
; CHECK-NEXT:    cmhi v5.2d, v16.2d, v5.2d
; CHECK-NEXT:    cmhi v6.2d, v16.2d, v6.2d
; CHECK-NEXT:    cset w9, lt
; CHECK-NEXT:    cmhi v0.2d, v16.2d, v0.2d
; CHECK-NEXT:    uzp1 v16.4s, v21.4s, v20.4s
; CHECK-NEXT:    cmp x10, #1
; CHECK-NEXT:    uzp1 v20.4s, v23.4s, v22.4s
; CHECK-NEXT:    uzp1 v17.4s, v17.4s, v24.4s
; CHECK-NEXT:    cset w10, lt
; CHECK-NEXT:    uzp1 v2.4s, v3.4s, v2.4s
; CHECK-NEXT:    uzp1 v3.4s, v19.4s, v7.4s
; CHECK-NEXT:    uzp1 v4.4s, v5.4s, v4.4s
; CHECK-NEXT:    uzp1 v5.4s, v18.4s, v6.4s
; CHECK-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
; CHECK-NEXT:    uzp1 v1.8h, v17.8h, v20.8h
; CHECK-NEXT:    uzp1 v3.8h, v16.8h, v3.8h
; CHECK-NEXT:    uzp1 v4.8h, v5.8h, v4.8h
; CHECK-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
; CHECK-NEXT:    dup v2.16b, w10
; CHECK-NEXT:    uzp1 v1.16b, v1.16b, v3.16b
; CHECK-NEXT:    dup v3.16b, w9
; CHECK-NEXT:    adrp x9, .LCPI18_0
; CHECK-NEXT:    uzp1 v0.16b, v4.16b, v0.16b
; CHECK-NEXT:    orr v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI18_0]
; CHECK-NEXT:    orr v0.16b, v0.16b, v3.16b
; CHECK-NEXT:    shl v1.16b, v1.16b, #7
; CHECK-NEXT:    shl v0.16b, v0.16b, #7
; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    ext v2.16b, v1.16b, v1.16b, #8
; CHECK-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
; CHECK-NEXT:    zip1 v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    zip1 v0.16b, v0.16b, v3.16b
; CHECK-NEXT:    addv h1, v1.8h
; CHECK-NEXT:    addv h0, v0.8h
; CHECK-NEXT:    str h1, [x8]
; CHECK-NEXT:    str h0, [x8, #2]
; CHECK-NEXT:    ret
entry:
  %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1(ptr %a, ptr %b, i64 8)
  ret <32 x i1> %0
}

define <9 x i1> @whilewr_8_widen(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_8_widen:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.b, x0, x1
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    umov w9, v0.b[0]
; CHECK-NEXT:    umov w10, v0.b[1]
; CHECK-NEXT:    umov w11, v0.b[2]
; CHECK-NEXT:    umov w12, v0.b[7]
; CHECK-NEXT:    and w9, w9, #0x1
; CHECK-NEXT:    bfi w9, w10, #1, #1
; CHECK-NEXT:    umov w10, v0.b[3]
; CHECK-NEXT:    bfi w9, w11, #2, #1
; CHECK-NEXT:    umov w11, v0.b[4]
; CHECK-NEXT:    bfi w9, w10, #3, #1
; CHECK-NEXT:    umov w10, v0.b[5]
; CHECK-NEXT:    bfi w9, w11, #4, #1
; CHECK-NEXT:    umov w11, v0.b[6]
; CHECK-NEXT:    bfi w9, w10, #5, #1
; CHECK-NEXT:    umov w10, v0.b[8]
; CHECK-NEXT:    bfi w9, w11, #6, #1
; CHECK-NEXT:    ubfiz w11, w12, #7, #1
; CHECK-NEXT:    orr w9, w9, w11
; CHECK-NEXT:    orr w9, w9, w10, lsl #8
; CHECK-NEXT:    and w9, w9, #0x1ff
; CHECK-NEXT:    strh w9, [x8]
; CHECK-NEXT:    ret
entry:
  %0 = call <9 x i1> @llvm.loop.dependence.war.mask.v9i1(ptr %a, ptr %b, i64 1)
  ret <9 x i1> %0
}

define <7 x i1> @whilewr_16_widen(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_16_widen:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.h, x0, x1
; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.8b, v0.8h
; CHECK-NEXT:    umov w0, v0.b[0]
; CHECK-NEXT:    umov w1, v0.b[1]
; CHECK-NEXT:    umov w2, v0.b[2]
; CHECK-NEXT:    umov w3, v0.b[3]
; CHECK-NEXT:    umov w4, v0.b[4]
; CHECK-NEXT:    umov w5, v0.b[5]
; CHECK-NEXT:    umov w6, v0.b[6]
; CHECK-NEXT:    ret
entry:
  %0 = call <7 x i1> @llvm.loop.dependence.war.mask.v7i1(ptr %a, ptr %b, i64 2)
  ret <7 x i1> %0
}

define <3 x i1> @whilewr_32_widen(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_32_widen:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.s, x0, x1
; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    umov w0, v0.h[0]
; CHECK-NEXT:    umov w1, v0.h[1]
; CHECK-NEXT:    umov w2, v0.h[2]
; CHECK-NEXT:    ret
entry:
  %0 = call <3 x i1> @llvm.loop.dependence.war.mask.v3i1(ptr %a, ptr %b, i64 4)
  ret <3 x i1> %0
}

define <16 x i1> @whilewr_badimm(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_badimm:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov x8, #6148914691236517205 // =0x5555555555555555
; CHECK-NEXT:    sub x9, x1, x0
; CHECK-NEXT:    index z0.d, #0, #1
; CHECK-NEXT:    movk x8, #21846
; CHECK-NEXT:    smulh x8, x9, x8
; CHECK-NEXT:    mov z1.d, z0.d
; CHECK-NEXT:    mov z2.d, z0.d
; CHECK-NEXT:    mov z4.d, z0.d
; CHECK-NEXT:    mov z5.d, z0.d
; CHECK-NEXT:    mov z6.d, z0.d
; CHECK-NEXT:    mov z7.d, z0.d
; CHECK-NEXT:    mov z16.d, z0.d
; CHECK-NEXT:    add x8, x8, x8, lsr #63
; CHECK-NEXT:    add z1.d, z1.d, #12 // =0xc
; CHECK-NEXT:    add z2.d, z2.d, #10 // =0xa
; CHECK-NEXT:    add z4.d, z4.d, #8 // =0x8
; CHECK-NEXT:    add z5.d, z5.d, #6 // =0x6
; CHECK-NEXT:    add z6.d, z6.d, #4 // =0x4
; CHECK-NEXT:    dup v3.2d, x8
; CHECK-NEXT:    add z16.d, z16.d, #14 // =0xe
; CHECK-NEXT:    add z7.d, z7.d, #2 // =0x2
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    cset w8, lt
; CHECK-NEXT:    cmhi v0.2d, v3.2d, v0.2d
; CHECK-NEXT:    cmhi v1.2d, v3.2d, v1.2d
; CHECK-NEXT:    cmhi v2.2d, v3.2d, v2.2d
; CHECK-NEXT:    cmhi v4.2d, v3.2d, v4.2d
; CHECK-NEXT:    cmhi v16.2d, v3.2d, v16.2d
; CHECK-NEXT:    cmhi v5.2d, v3.2d, v5.2d
; CHECK-NEXT:    cmhi v6.2d, v3.2d, v6.2d
; CHECK-NEXT:    cmhi v3.2d, v3.2d, v7.2d
; CHECK-NEXT:    uzp1 v1.4s, v1.4s, v16.4s
; CHECK-NEXT:    uzp1 v2.4s, v4.4s, v2.4s
; CHECK-NEXT:    uzp1 v4.4s, v6.4s, v5.4s
; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v3.4s
; CHECK-NEXT:    uzp1 v1.8h, v2.8h, v1.8h
; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    dup v1.16b, w8
; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1(ptr %a, ptr %b, i64 3)
  ret <16 x i1> %0
}

; Scalarizing <1 x i1> types

define <1 x i1> @whilewr_8_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_8_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    cmp x8, #0
; CHECK-NEXT:    cset w8, gt
; CHECK-NEXT:    cmp x1, x0
; CHECK-NEXT:    csinc w0, w8, wzr, ne
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1(ptr %a, ptr %b, i64 1)
  ret <1 x i1> %0
}

define <1 x i1> @whilewr_16_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_16_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    cset w8, gt
; CHECK-NEXT:    cmp x1, x0
; CHECK-NEXT:    csinc w0, w8, wzr, ne
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1(ptr %a, ptr %b, i64 2)
  ret <1 x i1> %0
}

define <1 x i1> @whilewr_32_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_32_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    cmp x8, #3
; CHECK-NEXT:    cset w8, gt
; CHECK-NEXT:    cmp x1, x0
; CHECK-NEXT:    csinc w0, w8, wzr, ne
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1(ptr %a, ptr %b, i64 4)
  ret <1 x i1> %0
}

define <1 x i1> @whilewr_64_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilewr_64_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    cmp x8, #7
; CHECK-NEXT:    cset w8, gt
; CHECK-NEXT:    cmp x1, x0
; CHECK-NEXT:    csinc w0, w8, wzr, ne
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1(ptr %a, ptr %b, i64 8)
  ret <1 x i1> %0
}

define <1 x i1> @whilerw_8_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_8_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    cmp x8, #0
; CHECK-NEXT:    cset w8, gt
; CHECK-NEXT:    cmp x1, x0
; CHECK-NEXT:    csinc w0, w8, wzr, ne
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1(ptr %a, ptr %b, i64 1)
  ret <1 x i1> %0
}

define <1 x i1> @whilerw_16_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_16_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    cset w8, gt
; CHECK-NEXT:    cmp x1, x0
; CHECK-NEXT:    csinc w0, w8, wzr, ne
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1(ptr %a, ptr %b, i64 2)
  ret <1 x i1> %0
}

define <1 x i1> @whilerw_32_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_32_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    cmp x8, #3
; CHECK-NEXT:    cset w8, gt
; CHECK-NEXT:    cmp x1, x0
; CHECK-NEXT:    csinc w0, w8, wzr, ne
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1(ptr %a, ptr %b, i64 4)
  ret <1 x i1> %0
}

define <1 x i1> @whilerw_64_scalarize(ptr %a, ptr %b) {
; CHECK-LABEL: whilerw_64_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    cmp x8, #7
; CHECK-NEXT:    cset w8, gt
; CHECK-NEXT:    cmp x1, x0
; CHECK-NEXT:    csinc w0, w8, wzr, ne
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1(ptr %a, ptr %b, i64 8)
  ret <1 x i1> %0
}