; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx906 < %s | FileCheck --check-prefix=GFX906 %s

define amdgpu_kernel void @v3i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
; GFX906-LABEL: v3i8_liveout:
; GFX906:       ; %bb.0: ; %entry
; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX906-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
; GFX906-NEXT:    v_mov_b32_e32 v3, 8
; GFX906-NEXT:    v_mov_b32_e32 v5, 16
; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
; GFX906-NEXT:    global_load_dword v4, v2, s[0:1]
; GFX906-NEXT:    v_mov_b32_e32 v1, 0xff
; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    v_and_b32_e32 v6, 0xff, v4
; GFX906-NEXT:    v_lshlrev_b32_sdwa v7, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
; GFX906-NEXT:    v_lshlrev_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX906-NEXT:    v_or3_b32 v4, v6, v7, v4
; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX906-NEXT:    s_cbranch_execz .LBB0_2
; GFX906-NEXT:  ; %bb.1: ; %bb.1
; GFX906-NEXT:    global_load_dword v0, v2, s[2:3]
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    v_and_b32_e32 v2, 0xff, v0
; GFX906-NEXT:    v_lshlrev_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
; GFX906-NEXT:    v_lshlrev_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX906-NEXT:    v_or3_b32 v4, v2, v3, v0
; GFX906-NEXT:  .LBB0_2: ; %bb.2
; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX906-NEXT:    v_lshrrev_b32_e32 v0, 8, v4
; GFX906-NEXT:    v_and_b32_e32 v0, 0xff, v0
; GFX906-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
; GFX906-NEXT:    v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX906-NEXT:    v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
; GFX906-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX906-NEXT:    v_mov_b32_e32 v1, 0
; GFX906-NEXT:    global_store_short v1, v0, s[6:7]
; GFX906-NEXT:    global_store_byte_d16_hi v1, v0, s[6:7] offset:2
; GFX906-NEXT:    s_endpgm
entry:
  %idx = call i32 @llvm.amdgcn.workitem.id.x()
  %gep1 = getelementptr <3 x i8>, ptr addrspace(1) %src1, i32 %idx
  %vec1 = load <3 x i8>, ptr addrspace(1) %gep1
  %gep2 = getelementptr <3 x i8>, ptr addrspace(1) %src2, i32 %idx
  %vec2 = load <3 x i8>, ptr addrspace(1) %gep2
  %cmp = icmp ult i32 %idx, 15
  br i1 %cmp, label %bb.1, label %bb.2
bb.1:
  br label %bb.2

bb.2:
  %tmp5 = phi <3 x i8> [ %vec1, %entry ], [ %vec2, %bb.1 ]
  store <3 x i8> %tmp5, ptr addrspace(1) %dst, align 4
  ret void
}

define amdgpu_kernel void @v4i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
; GFX906-LABEL: v4i8_liveout:
; GFX906:       ; %bb.0: ; %entry
; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX906-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
; GFX906-NEXT:    global_load_dword v1, v2, s[0:1]
; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX906-NEXT:    s_cbranch_execz .LBB1_2
; GFX906-NEXT:  ; %bb.1: ; %bb.1
; GFX906-NEXT:    global_load_dword v1, v2, s[2:3]
; GFX906-NEXT:  .LBB1_2: ; %bb.2
; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX906-NEXT:    v_mov_b32_e32 v0, 0
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    global_store_dword v0, v1, s[6:7]
; GFX906-NEXT:    s_endpgm
entry:
  %idx = call i32 @llvm.amdgcn.workitem.id.x()
  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
  %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
  %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
  %cmp = icmp ult i32 %idx, 15
  br i1 %cmp, label %bb.1, label %bb.2
bb.1:
  br label %bb.2

bb.2:
  %tmp5 = phi <4 x i8> [ %vec1, %entry ], [ %vec2, %bb.1 ]
  store <4 x i8> %tmp5, ptr addrspace(1) %dst, align 4
  ret void
}

define amdgpu_kernel void @v5i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
; GFX906-LABEL: v5i8_liveout:
; GFX906:       ; %bb.0: ; %entry
; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX906-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
; GFX906-NEXT:    global_load_dwordx2 v[1:2], v3, s[0:1]
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    v_and_b32_e32 v2, 0xff, v2
; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX906-NEXT:    s_cbranch_execz .LBB2_2
; GFX906-NEXT:  ; %bb.1: ; %bb.1
; GFX906-NEXT:    global_load_dwordx2 v[1:2], v3, s[2:3]
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    v_and_b32_e32 v2, 0xff, v2
; GFX906-NEXT:  .LBB2_2: ; %bb.2
; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX906-NEXT:    v_mov_b32_e32 v4, 0
; GFX906-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
; GFX906-NEXT:    v_lshrrev_b32_e32 v3, 24, v1
; GFX906-NEXT:    global_store_byte v4, v1, s[6:7]
; GFX906-NEXT:    global_store_byte v4, v0, s[6:7] offset:1
; GFX906-NEXT:    global_store_byte_d16_hi v4, v1, s[6:7] offset:2
; GFX906-NEXT:    global_store_byte v4, v3, s[6:7] offset:3
; GFX906-NEXT:    global_store_byte v4, v2, s[6:7] offset:4
; GFX906-NEXT:    s_endpgm
entry:
  %idx = call i32 @llvm.amdgcn.workitem.id.x()
  %gep1 = getelementptr <5 x i8>, ptr addrspace(1) %src1, i32 %idx
  %vec1 = load <5 x i8>, ptr addrspace(1) %gep1
  %gep2 = getelementptr <5 x i8>, ptr addrspace(1) %src2, i32 %idx
  %vec2 = load <5 x i8>, ptr addrspace(1) %gep2
  %cmp = icmp ult i32 %idx, 15
  br i1 %cmp, label %bb.1, label %bb.2
bb.1:
  br label %bb.2

bb.2:
  %tmp5 = phi <5 x i8> [ %vec1, %entry ], [ %vec2, %bb.1 ]
  store <5 x i8> %tmp5, ptr addrspace(1) %dst, align 4
  ret void
}

define amdgpu_kernel void @v8i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
; GFX906-LABEL: v8i8_liveout:
; GFX906:       ; %bb.0: ; %entry
; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX906-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
; GFX906-NEXT:    global_load_dwordx2 v[1:2], v3, s[0:1]
; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX906-NEXT:    s_cbranch_execz .LBB3_2
; GFX906-NEXT:  ; %bb.1: ; %bb.1
; GFX906-NEXT:    global_load_dwordx2 v[1:2], v3, s[2:3]
; GFX906-NEXT:  .LBB3_2: ; %bb.2
; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX906-NEXT:    v_mov_b32_e32 v0, 0
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    global_store_dwordx2 v0, v[1:2], s[6:7]
; GFX906-NEXT:    s_endpgm
entry:
  %idx = call i32 @llvm.amdgcn.workitem.id.x()
  %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
  %vec1 = load <8 x i8>, ptr addrspace(1) %gep1
  %gep2 = getelementptr <8 x i8>, ptr addrspace(1) %src2, i32 %idx
  %vec2 = load <8 x i8>, ptr addrspace(1) %gep2
  %cmp = icmp ult i32 %idx, 15
  br i1 %cmp, label %bb.1, label %bb.2
bb.1:
  br label %bb.2

bb.2:
  %tmp5 = phi <8 x i8> [ %vec1, %entry ], [ %vec2, %bb.1 ]
  store <8 x i8> %tmp5, ptr addrspace(1) %dst, align 4
  ret void
}

define amdgpu_kernel void @v16i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
; GFX906-LABEL: v16i8_liveout:
; GFX906:       ; %bb.0: ; %entry
; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX906-NEXT:    v_lshlrev_b32_e32 v5, 4, v0
; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
; GFX906-NEXT:    global_load_dwordx4 v[1:4], v5, s[0:1]
; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX906-NEXT:    s_cbranch_execz .LBB4_2
; GFX906-NEXT:  ; %bb.1: ; %bb.1
; GFX906-NEXT:    global_load_dwordx4 v[1:4], v5, s[2:3]
; GFX906-NEXT:  .LBB4_2: ; %bb.2
; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX906-NEXT:    v_mov_b32_e32 v0, 0
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    global_store_dwordx4 v0, v[1:4], s[6:7]
; GFX906-NEXT:    s_endpgm
entry:
  %idx = call i32 @llvm.amdgcn.workitem.id.x()
  %gep1 = getelementptr <16 x i8>, ptr addrspace(1) %src1, i32 %idx
  %vec1 = load <16 x i8>, ptr addrspace(1) %gep1
  %gep2 = getelementptr <16 x i8>, ptr addrspace(1) %src2, i32 %idx
  %vec2 = load <16 x i8>, ptr addrspace(1) %gep2
  %cmp = icmp ult i32 %idx, 15
  br i1 %cmp, label %bb.1, label %bb.2
bb.1:
  br label %bb.2

bb.2:
  %tmp5 = phi <16 x i8> [ %vec1, %entry ], [ %vec2, %bb.1 ]
  store <16 x i8> %tmp5, ptr addrspace(1) %dst, align 4
  ret void
}

define amdgpu_kernel void @v32i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
; GFX906-LABEL: v32i8_liveout:
; GFX906:       ; %bb.0: ; %entry
; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX906-NEXT:    v_lshlrev_b32_e32 v9, 5, v0
; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
; GFX906-NEXT:    global_load_dwordx4 v[1:4], v9, s[0:1]
; GFX906-NEXT:    global_load_dwordx4 v[5:8], v9, s[0:1] offset:16
; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX906-NEXT:    s_cbranch_execz .LBB5_2
; GFX906-NEXT:  ; %bb.1: ; %bb.1
; GFX906-NEXT:    global_load_dwordx4 v[1:4], v9, s[2:3]
; GFX906-NEXT:    global_load_dwordx4 v[5:8], v9, s[2:3] offset:16
; GFX906-NEXT:  .LBB5_2: ; %bb.2
; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX906-NEXT:    v_mov_b32_e32 v0, 0
; GFX906-NEXT:    s_waitcnt vmcnt(1)
; GFX906-NEXT:    global_store_dwordx4 v0, v[1:4], s[6:7]
; GFX906-NEXT:    s_waitcnt vmcnt(1)
; GFX906-NEXT:    global_store_dwordx4 v0, v[5:8], s[6:7] offset:16
; GFX906-NEXT:    s_endpgm
entry:
  %idx = call i32 @llvm.amdgcn.workitem.id.x()
  %gep1 = getelementptr <32 x i8>, ptr addrspace(1) %src1, i32 %idx
  %vec1 = load <32 x i8>, ptr addrspace(1) %gep1
  %gep2 = getelementptr <32 x i8>, ptr addrspace(1) %src2, i32 %idx
  %vec2 = load <32 x i8>, ptr addrspace(1) %gep2
  %cmp = icmp ult i32 %idx, 15
  br i1 %cmp, label %bb.1, label %bb.2
bb.1:
  br label %bb.2

bb.2:
  %tmp5 = phi <32 x i8> [ %vec1, %entry ], [ %vec2, %bb.1 ]
  store <32 x i8> %tmp5, ptr addrspace(1) %dst, align 4
  ret void
}

define amdgpu_kernel void @v256i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
; GFX906-LABEL: v256i8_liveout:
; GFX906:       ; %bb.0: ; %entry
; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX906-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
; GFX906-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX906-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
; GFX906-NEXT:    global_load_dwordx4 v[5:8], v4, s[0:1]
; GFX906-NEXT:    s_mov_b32 s14, -1
; GFX906-NEXT:    s_mov_b32 s15, 0xe00000
; GFX906-NEXT:    s_add_u32 s12, s12, s11
; GFX906-NEXT:    s_addc_u32 s13, s13, 0
; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    buffer_store_dword v5, off, s[12:15], 0 ; 4-byte Folded Spill
; GFX906-NEXT:    s_nop 0
; GFX906-NEXT:    buffer_store_dword v6, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
; GFX906-NEXT:    buffer_store_dword v7, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
; GFX906-NEXT:    buffer_store_dword v8, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
; GFX906-NEXT:    global_load_dwordx4 v[5:8], v4, s[0:1] offset:16
; GFX906-NEXT:    s_nop 0
; GFX906-NEXT:    global_load_dwordx4 v[9:12], v4, s[0:1] offset:32
; GFX906-NEXT:    global_load_dwordx4 v[13:16], v4, s[0:1] offset:48
; GFX906-NEXT:    global_load_dwordx4 v[17:20], v4, s[0:1] offset:64
; GFX906-NEXT:    global_load_dwordx4 v[21:24], v4, s[0:1] offset:80
; GFX906-NEXT:    global_load_dwordx4 v[25:28], v4, s[0:1] offset:96
; GFX906-NEXT:    global_load_dwordx4 v[29:32], v4, s[0:1] offset:112
; GFX906-NEXT:    global_load_dwordx4 v[33:36], v4, s[0:1] offset:128
; GFX906-NEXT:    global_load_dwordx4 v[37:40], v4, s[0:1] offset:144
; GFX906-NEXT:    global_load_dwordx4 v[41:44], v4, s[0:1] offset:160
; GFX906-NEXT:    global_load_dwordx4 v[45:48], v4, s[0:1] offset:176
; GFX906-NEXT:    global_load_dwordx4 v[49:52], v4, s[0:1] offset:192
; GFX906-NEXT:    global_load_dwordx4 v[53:56], v4, s[0:1] offset:208
; GFX906-NEXT:    global_load_dwordx4 v[57:60], v4, s[0:1] offset:224
; GFX906-NEXT:    global_load_dwordx4 v[0:3], v4, s[0:1] offset:240
; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX906-NEXT:    s_cbranch_execz .LBB6_2
; GFX906-NEXT:  ; %bb.1: ; %bb.1
; GFX906-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3]
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    buffer_store_dword v0, off, s[12:15], 0 ; 4-byte Folded Spill
; GFX906-NEXT:    s_nop 0
; GFX906-NEXT:    buffer_store_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
; GFX906-NEXT:    buffer_store_dword v2, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
; GFX906-NEXT:    buffer_store_dword v3, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
; GFX906-NEXT:    global_load_dwordx4 v[5:8], v4, s[2:3] offset:16
; GFX906-NEXT:    global_load_dwordx4 v[9:12], v4, s[2:3] offset:32
; GFX906-NEXT:    global_load_dwordx4 v[13:16], v4, s[2:3] offset:48
; GFX906-NEXT:    global_load_dwordx4 v[17:20], v4, s[2:3] offset:64
; GFX906-NEXT:    global_load_dwordx4 v[21:24], v4, s[2:3] offset:80
; GFX906-NEXT:    global_load_dwordx4 v[25:28], v4, s[2:3] offset:96
; GFX906-NEXT:    global_load_dwordx4 v[29:32], v4, s[2:3] offset:112
; GFX906-NEXT:    global_load_dwordx4 v[33:36], v4, s[2:3] offset:128
; GFX906-NEXT:    global_load_dwordx4 v[37:40], v4, s[2:3] offset:144
; GFX906-NEXT:    global_load_dwordx4 v[41:44], v4, s[2:3] offset:160
; GFX906-NEXT:    global_load_dwordx4 v[45:48], v4, s[2:3] offset:176
; GFX906-NEXT:    global_load_dwordx4 v[49:52], v4, s[2:3] offset:192
; GFX906-NEXT:    global_load_dwordx4 v[53:56], v4, s[2:3] offset:208
; GFX906-NEXT:    global_load_dwordx4 v[57:60], v4, s[2:3] offset:224
; GFX906-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:240
; GFX906-NEXT:  .LBB6_2: ; %bb.2
; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    buffer_store_dword v0, off, s[12:15], 0 offset:16 ; 4-byte Folded Spill
; GFX906-NEXT:    s_nop 0
; GFX906-NEXT:    buffer_store_dword v1, off, s[12:15], 0 offset:20 ; 4-byte Folded Spill
; GFX906-NEXT:    buffer_store_dword v2, off, s[12:15], 0 offset:24 ; 4-byte Folded Spill
; GFX906-NEXT:    buffer_store_dword v3, off, s[12:15], 0 offset:28 ; 4-byte Folded Spill
; GFX906-NEXT:    v_mov_b32_e32 v0, v57
; GFX906-NEXT:    v_mov_b32_e32 v1, v58
; GFX906-NEXT:    v_mov_b32_e32 v2, v59
; GFX906-NEXT:    v_mov_b32_e32 v3, v60
; GFX906-NEXT:    v_mov_b32_e32 v60, v56
; GFX906-NEXT:    v_mov_b32_e32 v59, v55
; GFX906-NEXT:    v_mov_b32_e32 v58, v54
; GFX906-NEXT:    v_mov_b32_e32 v57, v53
; GFX906-NEXT:    v_mov_b32_e32 v56, v52
; GFX906-NEXT:    v_mov_b32_e32 v55, v51
; GFX906-NEXT:    v_mov_b32_e32 v54, v50
; GFX906-NEXT:    v_mov_b32_e32 v53, v49
; GFX906-NEXT:    v_mov_b32_e32 v52, v48
; GFX906-NEXT:    v_mov_b32_e32 v51, v47
; GFX906-NEXT:    v_mov_b32_e32 v50, v46
; GFX906-NEXT:    v_mov_b32_e32 v49, v45
; GFX906-NEXT:    v_mov_b32_e32 v48, v44
; GFX906-NEXT:    v_mov_b32_e32 v47, v43
; GFX906-NEXT:    v_mov_b32_e32 v46, v42
; GFX906-NEXT:    v_mov_b32_e32 v45, v41
; GFX906-NEXT:    v_mov_b32_e32 v44, v40
; GFX906-NEXT:    v_mov_b32_e32 v43, v39
; GFX906-NEXT:    v_mov_b32_e32 v42, v38
; GFX906-NEXT:    v_mov_b32_e32 v41, v37
; GFX906-NEXT:    v_mov_b32_e32 v40, v36
; GFX906-NEXT:    v_mov_b32_e32 v39, v35
; GFX906-NEXT:    v_mov_b32_e32 v38, v34
; GFX906-NEXT:    v_mov_b32_e32 v37, v33
; GFX906-NEXT:    v_mov_b32_e32 v36, v32
; GFX906-NEXT:    v_mov_b32_e32 v35, v31
; GFX906-NEXT:    v_mov_b32_e32 v34, v30
; GFX906-NEXT:    v_mov_b32_e32 v33, v29
; GFX906-NEXT:    v_mov_b32_e32 v32, v28
; GFX906-NEXT:    v_mov_b32_e32 v31, v27
; GFX906-NEXT:    v_mov_b32_e32 v30, v26
; GFX906-NEXT:    v_mov_b32_e32 v29, v25
; GFX906-NEXT:    v_mov_b32_e32 v28, v24
; GFX906-NEXT:    v_mov_b32_e32 v27, v23
; GFX906-NEXT:    v_mov_b32_e32 v26, v22
; GFX906-NEXT:    v_mov_b32_e32 v25, v21
; GFX906-NEXT:    v_mov_b32_e32 v24, v20
; GFX906-NEXT:    v_mov_b32_e32 v23, v19
; GFX906-NEXT:    v_mov_b32_e32 v22, v18
; GFX906-NEXT:    v_mov_b32_e32 v21, v17
; GFX906-NEXT:    v_mov_b32_e32 v20, v16
; GFX906-NEXT:    v_mov_b32_e32 v19, v15
; GFX906-NEXT:    v_mov_b32_e32 v18, v14
; GFX906-NEXT:    v_mov_b32_e32 v17, v13
; GFX906-NEXT:    v_mov_b32_e32 v16, v12
; GFX906-NEXT:    v_mov_b32_e32 v15, v11
; GFX906-NEXT:    v_mov_b32_e32 v14, v10
; GFX906-NEXT:    v_mov_b32_e32 v13, v9
; GFX906-NEXT:    v_mov_b32_e32 v12, v8
; GFX906-NEXT:    v_mov_b32_e32 v11, v7
; GFX906-NEXT:    v_mov_b32_e32 v10, v6
; GFX906-NEXT:    v_mov_b32_e32 v9, v5
; GFX906-NEXT:    buffer_load_dword v5, off, s[12:15], 0 ; 4-byte Folded Reload
; GFX906-NEXT:    buffer_load_dword v6, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GFX906-NEXT:    buffer_load_dword v7, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
; GFX906-NEXT:    buffer_load_dword v8, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
; GFX906-NEXT:    v_mov_b32_e32 v4, 0
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    global_store_dwordx4 v4, v[5:8], s[6:7]
; GFX906-NEXT:    global_store_dwordx4 v4, v[9:12], s[6:7] offset:16
; GFX906-NEXT:    global_store_dwordx4 v4, v[13:16], s[6:7] offset:32
; GFX906-NEXT:    global_store_dwordx4 v4, v[17:20], s[6:7] offset:48
; GFX906-NEXT:    global_store_dwordx4 v4, v[21:24], s[6:7] offset:64
; GFX906-NEXT:    global_store_dwordx4 v4, v[25:28], s[6:7] offset:80
; GFX906-NEXT:    global_store_dwordx4 v4, v[29:32], s[6:7] offset:96
; GFX906-NEXT:    global_store_dwordx4 v4, v[33:36], s[6:7] offset:112
; GFX906-NEXT:    global_store_dwordx4 v4, v[37:40], s[6:7] offset:128
; GFX906-NEXT:    global_store_dwordx4 v4, v[41:44], s[6:7] offset:144
; GFX906-NEXT:    global_store_dwordx4 v4, v[45:48], s[6:7] offset:160
; GFX906-NEXT:    global_store_dwordx4 v4, v[49:52], s[6:7] offset:176
; GFX906-NEXT:    global_store_dwordx4 v4, v[53:56], s[6:7] offset:192
; GFX906-NEXT:    global_store_dwordx4 v4, v[57:60], s[6:7] offset:208
; GFX906-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:224
; GFX906-NEXT:    buffer_load_dword v0, off, s[12:15], 0 offset:16 ; 4-byte Folded Reload
; GFX906-NEXT:    s_nop 0
; GFX906-NEXT:    buffer_load_dword v1, off, s[12:15], 0 offset:20 ; 4-byte Folded Reload
; GFX906-NEXT:    buffer_load_dword v2, off, s[12:15], 0 offset:24 ; 4-byte Folded Reload
; GFX906-NEXT:    buffer_load_dword v3, off, s[12:15], 0 offset:28 ; 4-byte Folded Reload
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:240
; GFX906-NEXT:    s_endpgm
entry:
  %idx = call i32 @llvm.amdgcn.workitem.id.x()
  %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
  %vec1 = load <256 x i8>, ptr addrspace(1) %gep1
  %gep2 = getelementptr <8 x i8>, ptr addrspace(1) %src2, i32 %idx
  %vec2 = load <256 x i8>, ptr addrspace(1) %gep2
  %cmp = icmp ult i32 %idx, 15
  br i1 %cmp, label %bb.1, label %bb.2
bb.1:
  br label %bb.2

bb.2:
  %tmp5 = phi <256 x i8> [ %vec1, %entry ], [ %vec2, %bb.1 ]
  store <256 x i8> %tmp5, ptr addrspace(1) %dst, align 4
  ret void
}


define amdgpu_kernel void @repeat_successor(i32 %in, ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
; GFX906-LABEL: repeat_successor:
; GFX906:       ; %bb.0: ; %entry
; GFX906-NEXT:    s_load_dword s6, s[4:5], 0x24
; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
; GFX906-NEXT:    s_cmp_lt_i32 s6, 3
; GFX906-NEXT:    s_cbranch_scc0 .LBB7_3
; GFX906-NEXT:  ; %bb.1: ; %LeafBlock
; GFX906-NEXT:    s_cmp_ge_i32 s6, 1
; GFX906-NEXT:    s_cbranch_scc0 .LBB7_6
; GFX906-NEXT:  ; %bb.2:
; GFX906-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX906-NEXT:    global_load_dword v0, v0, s[0:1]
; GFX906-NEXT:    s_branch .LBB7_5
; GFX906-NEXT:  .LBB7_3: ; %LeafBlock5
; GFX906-NEXT:    s_cmp_eq_u32 s6, 3
; GFX906-NEXT:    s_cbranch_scc0 .LBB7_6
; GFX906-NEXT:  ; %bb.4: ; %sw.bb5
; GFX906-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX906-NEXT:    global_load_dword v0, v0, s[2:3]
; GFX906-NEXT:  .LBB7_5: ; %return.sink.split
; GFX906-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x3c
; GFX906-NEXT:    v_mov_b32_e32 v1, 0
; GFX906-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX906-NEXT:    global_store_dword v1, v0, s[0:1]
; GFX906-NEXT:  .LBB7_6: ; %return
; GFX906-NEXT:    s_endpgm
entry:
  %idx = call i32 @llvm.amdgcn.workitem.id.x()
  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
  %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
  %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
  switch i32 %in, label %return [
    i32 1, label %return.sink.split
    i32 2, label %return.sink.split
    i32 3, label %sw.bb5
  ]

sw.bb5:
  br label %return.sink.split

return.sink.split:
  %tmp5 = phi <4 x i8> [ %vec2, %sw.bb5 ], [ %vec1, %entry ], [ %vec1, %entry ]
  store <4 x i8> %tmp5, ptr addrspace(1) %dst, align 4
  ret void

return:
  ret void
}

define amdgpu_kernel void @v8i8_phi_chain(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst0, ptr addrspace(1) nocapture %dst1) {
; GFX906-LABEL: v8i8_phi_chain:
; GFX906:       ; %bb.0: ; %entry
; GFX906-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX906-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
; GFX906-NEXT:    s_xor_b64 s[0:1], vcc, -1
; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
; GFX906-NEXT:    global_load_dwordx2 v[1:2], v3, s[8:9]
; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX906-NEXT:    s_cbranch_execz .LBB8_2
; GFX906-NEXT:  ; %bb.1: ; %bb.1
; GFX906-NEXT:    global_load_dwordx2 v[1:2], v3, s[10:11]
; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v0
; GFX906-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
; GFX906-NEXT:    s_and_b64 s[4:5], exec, vcc
; GFX906-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX906-NEXT:  .LBB8_2: ; %Flow
; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
; GFX906-NEXT:    s_cbranch_execz .LBB8_4
; GFX906-NEXT:  ; %bb.3: ; %bb.2
; GFX906-NEXT:    v_mov_b32_e32 v0, 0
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    global_store_dwordx2 v0, v[1:2], s[12:13]
; GFX906-NEXT:  .LBB8_4: ; %bb.3
; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX906-NEXT:    v_mov_b32_e32 v0, 0
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    global_store_dwordx2 v0, v[1:2], s[14:15]
; GFX906-NEXT:    s_endpgm
entry:
  %idx = call i32 @llvm.amdgcn.workitem.id.x()
  %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
  %vec1 = load <8 x i8>, ptr addrspace(1) %gep1
  %gep2 = getelementptr <8 x i8>, ptr addrspace(1) %src2, i32 %idx
  %vec2 = load <8 x i8>, ptr addrspace(1) %gep2
  %cmp = icmp ult i32 %idx, 15
  br i1 %cmp, label %bb.1, label %bb.2
bb.1:
  %cmp2 = icmp ult i32 %idx, 7
  br i1 %cmp2, label %bb.2, label %bb.3

bb.2:
  %tmp5 = phi <8 x i8> [ %vec1, %entry ], [ %vec2, %bb.1 ]
  store <8 x i8> %tmp5, ptr addrspace(1) %dst0, align 4
  br label %bb.3

bb.3:
  %tmp7 = phi <8 x i8> [ %vec2, %bb.1], [%tmp5, %bb.2]
  store <8 x i8> %tmp7, ptr addrspace(1) %dst1, align 4
  ret void
}

define amdgpu_kernel void @v8i8_multi_block(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst0, ptr addrspace(1) nocapture %dst1) {
; GFX906-LABEL: v8i8_multi_block:
; GFX906:       ; %bb.0: ; %entry
; GFX906-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX906-NEXT:    v_lshlrev_b32_e32 v5, 3, v0
; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
; GFX906-NEXT:    global_load_dwordx2 v[3:4], v5, s[8:9]
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    v_mov_b32_e32 v1, v3
; GFX906-NEXT:    v_mov_b32_e32 v2, v4
; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX906-NEXT:    s_cbranch_execz .LBB9_4
; GFX906-NEXT:  ; %bb.1: ; %bb.1
; GFX906-NEXT:    global_load_dwordx2 v[1:2], v5, s[10:11]
; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v0
; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX906-NEXT:    s_cbranch_execz .LBB9_3
; GFX906-NEXT:  ; %bb.2: ; %bb.2
; GFX906-NEXT:    v_mov_b32_e32 v0, 0
; GFX906-NEXT:    global_store_dwordx2 v0, v[3:4], s[12:13]
; GFX906-NEXT:  .LBB9_3: ; %Flow
; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX906-NEXT:  .LBB9_4: ; %bb.3
; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX906-NEXT:    v_mov_b32_e32 v0, 0
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    global_store_dwordx2 v0, v[1:2], s[14:15]
; GFX906-NEXT:    s_endpgm
entry:
  %idx = call i32 @llvm.amdgcn.workitem.id.x()
  %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
  %vec1 = load <8 x i8>, ptr addrspace(1) %gep1
  %gep2 = getelementptr <8 x i8>, ptr addrspace(1) %src2, i32 %idx
  %vec2 = load <8 x i8>, ptr addrspace(1) %gep2
  %cmp = icmp ult i32 %idx, 15
  br i1 %cmp, label %bb.1, label %bb.3
bb.1:
  %cmp2 = icmp ult i32 %idx, 7
  br i1 %cmp2, label %bb.2, label %bb.3

bb.2:
  store <8 x i8> %vec1, ptr addrspace(1) %dst0, align 4
  br label %bb.3

bb.3:
  %tmp5 = phi <8 x i8> [ %vec1, %entry ], [ %vec2, %bb.1 ], [ %vec2, %bb.2]
  store <8 x i8> %tmp5, ptr addrspace(1) %dst1, align 4
  ret void
}

define amdgpu_kernel void @v32i8_loop_carried(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
; GFX906-LABEL: v32i8_loop_carried:
; GFX906:       ; %bb.0: ; %entry
; GFX906-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX906-NEXT:    v_lshlrev_b32_e32 v1, 5, v0
; GFX906-NEXT:    v_mov_b32_e32 v3, 8
; GFX906-NEXT:    v_mov_b32_e32 v2, 0xff
; GFX906-NEXT:    v_cmp_le_u32_e32 vcc, 15, v0
; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
; GFX906-NEXT:    global_load_dword v1, v1, s[0:1]
; GFX906-NEXT:    s_mov_b64 s[0:1], 0
; GFX906-NEXT:    s_waitcnt vmcnt(0)
; GFX906-NEXT:    v_lshlrev_b32_sdwa v0, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX906-NEXT:    v_and_or_b32 v0, v1, v2, v0
; GFX906-NEXT:    v_mov_b32_e32 v2, 24
; GFX906-NEXT:  .LBB10_1: ; %bb.1
; GFX906-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX906-NEXT:    v_and_b32_e32 v3, 0xff, v1
; GFX906-NEXT:    s_and_b64 s[2:3], exec, vcc
; GFX906-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
; GFX906-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX906-NEXT:    s_or_b64 s[0:1], s[2:3], s[0:1]
; GFX906-NEXT:    v_or3_b32 v1, v0, v3, v1
; GFX906-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX906-NEXT:    s_cbranch_execnz .LBB10_1
; GFX906-NEXT:  ; %bb.2: ; %bb.2.loopexit
; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX906-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX906-NEXT:    v_mov_b32_e32 v0, 0
; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
; GFX906-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX906-NEXT:    s_endpgm
entry:
  %idx = call i32 @llvm.amdgcn.workitem.id.x()
  %gep1 = getelementptr <32 x i8>, ptr addrspace(1) %src1, i32 %idx
  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
  br label %bb.1

bb.1:
  %temp = phi <4 x i8> [ %vec1, %entry ], [ %vec2, %bb.1 ]
  %vec2 = shufflevector <4 x i8> %vec1, <4 x i8> %temp, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
  %cmp = icmp ult i32 %idx, 15
  br i1 %cmp, label %bb.1, label %bb.2
  br label %bb.2

bb.2:
  store <4 x i8> %vec2, ptr addrspace(1) %dst, align 4
  ret void
}


declare i32 @llvm.amdgcn.workitem.id.x()