path: root/llvm/test/CodeGen/AMDGPU/stack-realign.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji < %s | FileCheck -check-prefix=GCN %s

; Check that we properly realign the stack. While 4-byte access is all
; that is ever needed, some transformations rely on the known bits from the alignment of the pointer (e.g.


; 128 byte object
; 4 byte emergency stack slot
; = 144 bytes with padding between them

define void @needs_align16_default_stack_align(i32 %idx) #0 {
; GCN-LABEL: needs_align16_default_stack_align:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
; GCN-NEXT:    v_lshrrev_b32_e64 v2, 6, s32
; GCN-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
; GCN-NEXT:    v_mov_b32_e32 v2, 1
; GCN-NEXT:    v_mov_b32_e32 v1, 4
; GCN-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    v_or_b32_e32 v2, 12, v0
; GCN-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    v_or_b32_e32 v1, 8, v0
; GCN-NEXT:    v_mov_b32_e32 v2, 3
; GCN-NEXT:    buffer_store_dword v2, v1, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    v_or_b32_e32 v0, 4, v0
; GCN-NEXT:    v_mov_b32_e32 v1, 2
; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    s_setpc_b64 s[30:31]
; GCN: ; ScratchSize: 144
  %alloca.align16 = alloca [8 x <4 x i32>], align 16, addrspace(5)
  %gep0 = getelementptr inbounds [8 x <4 x i32>], ptr addrspace(5) %alloca.align16, i32 0, i32 %idx
  store volatile <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr addrspace(5) %gep0, align 16
  ret void
}

define void @needs_align16_stack_align4(i32 %idx) #2 {
; GCN-LABEL: needs_align16_stack_align4:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT:    s_mov_b32 s4, s33
; GCN-NEXT:    s_add_i32 s33, s32, 0x3c0
; GCN-NEXT:    s_and_b32 s33, s33, 0xfffffc00
; GCN-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
; GCN-NEXT:    v_lshrrev_b32_e64 v2, 6, s33
; GCN-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
; GCN-NEXT:    v_mov_b32_e32 v2, 1
; GCN-NEXT:    v_mov_b32_e32 v1, 4
; GCN-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    v_or_b32_e32 v2, 12, v0
; GCN-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    v_or_b32_e32 v1, 8, v0
; GCN-NEXT:    v_mov_b32_e32 v2, 3
; GCN-NEXT:    s_mov_b32 s5, s34
; GCN-NEXT:    s_mov_b32 s34, s32
; GCN-NEXT:    s_addk_i32 s32, 0x2800
; GCN-NEXT:    buffer_store_dword v2, v1, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    v_or_b32_e32 v0, 4, v0
; GCN-NEXT:    v_mov_b32_e32 v1, 2
; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    s_mov_b32 s32, s34
; GCN-NEXT:    s_mov_b32 s34, s5
; GCN-NEXT:    s_mov_b32 s33, s4
; GCN-NEXT:    s_setpc_b64 s[30:31]
; GCN: ; ScratchSize: 160
  %alloca.align16 = alloca [8 x <4 x i32>], align 16, addrspace(5)
  %gep0 = getelementptr inbounds [8 x <4 x i32>], ptr addrspace(5) %alloca.align16, i32 0, i32 %idx
  store volatile <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr addrspace(5) %gep0, align 16
  ret void
}


define void @needs_align32(i32 %idx) #0 {
; GCN-LABEL: needs_align32:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT:    s_mov_b32 s4, s33
; GCN-NEXT:    s_add_i32 s33, s32, 0x7c0
; GCN-NEXT:    s_and_b32 s33, s33, 0xfffff800
; GCN-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
; GCN-NEXT:    v_lshrrev_b32_e64 v2, 6, s33
; GCN-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
; GCN-NEXT:    v_mov_b32_e32 v2, 1
; GCN-NEXT:    v_mov_b32_e32 v1, 4
; GCN-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    v_or_b32_e32 v2, 12, v0
; GCN-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    v_or_b32_e32 v1, 8, v0
; GCN-NEXT:    v_mov_b32_e32 v2, 3
; GCN-NEXT:    s_mov_b32 s5, s34
; GCN-NEXT:    s_mov_b32 s34, s32
; GCN-NEXT:    s_addk_i32 s32, 0x3000
; GCN-NEXT:    buffer_store_dword v2, v1, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    v_or_b32_e32 v0, 4, v0
; GCN-NEXT:    v_mov_b32_e32 v1, 2
; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    s_mov_b32 s32, s34
; GCN-NEXT:    s_mov_b32 s34, s5
; GCN-NEXT:    s_mov_b32 s33, s4
; GCN-NEXT:    s_setpc_b64 s[30:31]
; GCN: ; ScratchSize: 192
  %alloca.align16 = alloca [8 x <4 x i32>], align 32, addrspace(5)
  %gep0 = getelementptr inbounds [8 x <4 x i32>], ptr addrspace(5) %alloca.align16, i32 0, i32 %idx
  store volatile <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr addrspace(5) %gep0, align 32
  ret void
}

define void @force_realign4(i32 %idx) #1 {
; GCN-LABEL: force_realign4:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT:    s_mov_b32 s4, s33
; GCN-NEXT:    s_add_i32 s33, s32, 0xc0
; GCN-NEXT:    s_and_b32 s33, s33, 0xffffff00
; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT:    v_lshrrev_b32_e64 v1, 6, s33
; GCN-NEXT:    s_mov_b32 s5, s34
; GCN-NEXT:    s_mov_b32 s34, s32
; GCN-NEXT:    s_addk_i32 s32, 0xd00
; GCN-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
; GCN-NEXT:    v_mov_b32_e32 v1, 3
; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    s_mov_b32 s32, s34
; GCN-NEXT:    s_mov_b32 s34, s5
; GCN-NEXT:    s_mov_b32 s33, s4
; GCN-NEXT:    s_setpc_b64 s[30:31]
; GCN: ; ScratchSize: 52
  %alloca.align16 = alloca [8 x i32], align 4, addrspace(5)
  %gep0 = getelementptr inbounds [8 x i32], ptr addrspace(5) %alloca.align16, i32 0, i32 %idx
  store volatile i32 3, ptr addrspace(5) %gep0, align 4
  ret void
}

define amdgpu_kernel void @kernel_call_align16_from_8() #0 {
; GCN-LABEL: kernel_call_align16_from_8:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_add_i32 s12, s12, s17
; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT:    s_add_u32 s0, s0, s17
; GCN-NEXT:    s_addc_u32 s1, s1, 0
; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GCN-NEXT:    s_mov_b32 s13, s15
; GCN-NEXT:    s_mov_b32 s12, s14
; GCN-NEXT:    s_getpc_b64 s[14:15]
; GCN-NEXT:    s_add_u32 s14, s14, needs_align16_default_stack_align@gotpcrel32@lo+4
; GCN-NEXT:    s_addc_u32 s15, s15, needs_align16_default_stack_align@gotpcrel32@hi+12
; GCN-NEXT:    s_load_dwordx2 s[18:19], s[14:15], 0x0
; GCN-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
; GCN-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
; GCN-NEXT:    v_mov_b32_e32 v3, 2
; GCN-NEXT:    v_or_b32_e32 v31, v0, v2
; GCN-NEXT:    s_mov_b32 s14, s16
; GCN-NEXT:    v_mov_b32_e32 v0, 1
; GCN-NEXT:    s_movk_i32 s32, 0x400
; GCN-NEXT:    buffer_store_dword v3, off, s[0:3], 0
; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT:    s_swappc_b64 s[30:31], s[18:19]
; GCN-NEXT:    s_endpgm
  %alloca = alloca i32, align 4, addrspace(5)
  store volatile i32 2, ptr addrspace(5) %alloca
  call void @needs_align16_default_stack_align(i32 1)
  ret void
}

; The call sequence should keep the stack on call aligned to 4
define amdgpu_kernel void @kernel_call_align16_from_5() {
; GCN-LABEL: kernel_call_align16_from_5:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_add_i32 s12, s12, s17
; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT:    s_add_u32 s0, s0, s17
; GCN-NEXT:    s_addc_u32 s1, s1, 0
; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GCN-NEXT:    s_mov_b32 s13, s15
; GCN-NEXT:    s_mov_b32 s12, s14
; GCN-NEXT:    s_getpc_b64 s[14:15]
; GCN-NEXT:    s_add_u32 s14, s14, needs_align16_default_stack_align@gotpcrel32@lo+4
; GCN-NEXT:    s_addc_u32 s15, s15, needs_align16_default_stack_align@gotpcrel32@hi+12
; GCN-NEXT:    s_load_dwordx2 s[18:19], s[14:15], 0x0
; GCN-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
; GCN-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
; GCN-NEXT:    v_mov_b32_e32 v3, 2
; GCN-NEXT:    v_or_b32_e32 v31, v0, v2
; GCN-NEXT:    s_mov_b32 s14, s16
; GCN-NEXT:    v_mov_b32_e32 v0, 1
; GCN-NEXT:    s_movk_i32 s32, 0x400
; GCN-NEXT:    buffer_store_byte v3, off, s[0:3], 0
; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT:    s_swappc_b64 s[30:31], s[18:19]
; GCN-NEXT:    s_endpgm
  %alloca0 = alloca i8, align 1, addrspace(5)
  store volatile i8 2, ptr  addrspace(5) %alloca0

  call void @needs_align16_default_stack_align(i32 1)
  ret void
}

define amdgpu_kernel void @kernel_call_align4_from_5() {
; GCN-LABEL: kernel_call_align4_from_5:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_add_i32 s12, s12, s17
; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT:    s_add_u32 s0, s0, s17
; GCN-NEXT:    s_addc_u32 s1, s1, 0
; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GCN-NEXT:    s_mov_b32 s13, s15
; GCN-NEXT:    s_mov_b32 s12, s14
; GCN-NEXT:    s_getpc_b64 s[14:15]
; GCN-NEXT:    s_add_u32 s14, s14, needs_align16_stack_align4@gotpcrel32@lo+4
; GCN-NEXT:    s_addc_u32 s15, s15, needs_align16_stack_align4@gotpcrel32@hi+12
; GCN-NEXT:    s_load_dwordx2 s[18:19], s[14:15], 0x0
; GCN-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
; GCN-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
; GCN-NEXT:    v_mov_b32_e32 v3, 2
; GCN-NEXT:    v_or_b32_e32 v31, v0, v2
; GCN-NEXT:    s_mov_b32 s14, s16
; GCN-NEXT:    v_mov_b32_e32 v0, 1
; GCN-NEXT:    s_movk_i32 s32, 0x400
; GCN-NEXT:    buffer_store_byte v3, off, s[0:3], 0
; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT:    s_swappc_b64 s[30:31], s[18:19]
; GCN-NEXT:    s_endpgm
  %alloca0 = alloca i8, align 1, addrspace(5)
  store volatile i8 2, ptr  addrspace(5) %alloca0

  call void @needs_align16_stack_align4(i32 1)
  ret void
}

define void @default_realign_align128(i32 %idx) #0 {
; GCN-LABEL: default_realign_align128:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT:    s_mov_b32 s4, s33
; GCN-NEXT:    s_add_i32 s33, s32, 0x1fc0
; GCN-NEXT:    s_and_b32 s33, s33, 0xffffe000
; GCN-NEXT:    s_mov_b32 s5, s34
; GCN-NEXT:    s_mov_b32 s34, s32
; GCN-NEXT:    s_addk_i32 s32, 0x4000
; GCN-NEXT:    v_mov_b32_e32 v0, 9
; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s33
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    s_mov_b32 s32, s34
; GCN-NEXT:    s_mov_b32 s34, s5
; GCN-NEXT:    s_mov_b32 s33, s4
; GCN-NEXT:    s_setpc_b64 s[30:31]
  %alloca.align = alloca i32, align 128, addrspace(5)
  store volatile i32 9, ptr addrspace(5) %alloca.align, align 128
  ret void
}

define void @disable_realign_align128(i32 %idx) #3 {
; GCN-LABEL: disable_realign_align128:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT:    v_mov_b32_e32 v0, 9
; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    s_setpc_b64 s[30:31]
  %alloca.align = alloca i32, align 128, addrspace(5)
  store volatile i32 9, ptr addrspace(5) %alloca.align, align 128
  ret void
}

declare void @extern_func(<32 x i32>, i32) #0
define void @func_call_align1024_bp_gets_vgpr_spill(<32 x i32> %a, i32 %b) #0 {
; The test forces the stack to be realigned to a new boundary
; since there is a local object with an alignment of 1024.
; Should use BP to access the incoming stack arguments.
; The BP value is saved/restored with a VGPR spill.
; GCN-LABEL: func_call_align1024_bp_gets_vgpr_spill:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT:    s_mov_b32 s16, s33
; GCN-NEXT:    s_add_i32 s33, s32, 0xffc0
; GCN-NEXT:    s_and_b32 s33, s33, 0xffff0000
; GCN-NEXT:    s_or_saveexec_b64 s[18:19], -1
; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:1028 ; 4-byte Folded Spill
; GCN-NEXT:    s_mov_b64 exec, s[18:19]
; GCN-NEXT:    v_writelane_b32 v40, s16, 2
; GCN-NEXT:    v_mov_b32_e32 v32, 0
; GCN-NEXT:    v_writelane_b32 v40, s34, 3
; GCN-NEXT:    s_mov_b32 s34, s32
; GCN-NEXT:    buffer_store_dword v32, off, s[0:3], s33 offset:1024
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s34
; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s34 offset:4
; GCN-NEXT:    s_add_i32 s32, s32, 0x30000
; GCN-NEXT:    s_getpc_b64 s[16:17]
; GCN-NEXT:    s_add_u32 s16, s16, extern_func@gotpcrel32@lo+4
; GCN-NEXT:    s_addc_u32 s17, s17, extern_func@gotpcrel32@hi+12
; GCN-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
; GCN-NEXT:    v_writelane_b32 v40, s30, 0
; GCN-NEXT:    v_writelane_b32 v40, s31, 1
; GCN-NEXT:    s_waitcnt vmcnt(1)
; GCN-NEXT:    buffer_store_dword v32, off, s[0:3], s32
; GCN-NEXT:    s_waitcnt vmcnt(1)
; GCN-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:4
; GCN-NEXT:    s_waitcnt lgkmcnt(0)
; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
; GCN-NEXT:    v_readlane_b32 s31, v40, 1
; GCN-NEXT:    v_readlane_b32 s30, v40, 0
; GCN-NEXT:    s_mov_b32 s32, s34
; GCN-NEXT:    v_readlane_b32 s4, v40, 2
; GCN-NEXT:    v_readlane_b32 s34, v40, 3
; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s33 offset:1028 ; 4-byte Folded Reload
; GCN-NEXT:    s_mov_b64 exec, s[6:7]
; GCN-NEXT:    s_mov_b32 s33, s4
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    s_setpc_b64 s[30:31]

  %temp = alloca i32, align 1024, addrspace(5)
  store volatile i32 0, ptr addrspace(5) %temp, align 1024
  call void @extern_func(<32 x i32> %a, i32 %b)
  ret void
}

%struct.Data = type { [9 x i32] }
define i32 @needs_align1024_stack_args_used_inside_loop(ptr addrspace(5) nocapture readonly byval(%struct.Data) align 8 %arg) local_unnamed_addr #4 {
; The local object allocation needed an alignment of 1024.
; Since the function argument is accessed in a loop with an
; index variable, the base pointer first get loaded into a VGPR
; and that value should be further referenced to load the incoming values.
; The BP value will get saved/restored in an SGPR at the prolgoue/epilogue.
; GCN-LABEL: needs_align1024_stack_args_used_inside_loop:
; GCN:       ; %bb.0: ; %begin
; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT:    s_mov_b32 s11, s33
; GCN-NEXT:    s_add_i32 s33, s32, 0xffc0
; GCN-NEXT:    s_mov_b32 s14, s34
; GCN-NEXT:    s_mov_b32 s34, s32
; GCN-NEXT:    s_and_b32 s33, s33, 0xffff0000
; GCN-NEXT:    v_lshrrev_b32_e64 v1, 6, s34
; GCN-NEXT:    v_mov_b32_e32 v0, 0
; GCN-NEXT:    s_mov_b32 s10, 0
; GCN-NEXT:    s_mov_b64 s[4:5], 0
; GCN-NEXT:    s_add_i32 s32, s32, 0x30000
; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:1024
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    ; implicit-def: $sgpr6_sgpr7
; GCN-NEXT:    s_branch .LBB10_2
; GCN-NEXT:  .LBB10_1: ; %Flow
; GCN-NEXT:    ; in Loop: Header=BB10_2 Depth=1
; GCN-NEXT:    s_or_b64 exec, exec, s[8:9]
; GCN-NEXT:    s_and_b64 s[8:9], exec, s[6:7]
; GCN-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
; GCN-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN-NEXT:    s_cbranch_execz .LBB10_4
; GCN-NEXT:  .LBB10_2: ; %loop_body
; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN-NEXT:    buffer_load_dword v0, v1, s[0:3], 0 offen
; GCN-NEXT:    s_or_b64 s[6:7], s[6:7], exec
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s10, v0
; GCN-NEXT:    v_mov_b32_e32 v0, 0
; GCN-NEXT:    s_and_saveexec_b64 s[8:9], vcc
; GCN-NEXT:    s_cbranch_execz .LBB10_1
; GCN-NEXT:  ; %bb.3: ; %loop_end
; GCN-NEXT:    ; in Loop: Header=BB10_2 Depth=1
; GCN-NEXT:    s_add_i32 s10, s10, 1
; GCN-NEXT:    s_cmp_eq_u32 s10, 9
; GCN-NEXT:    s_cselect_b64 s[12:13], -1, 0
; GCN-NEXT:    s_andn2_b64 s[6:7], s[6:7], exec
; GCN-NEXT:    s_and_b64 s[12:13], s[12:13], exec
; GCN-NEXT:    v_add_u32_e32 v1, vcc, 4, v1
; GCN-NEXT:    v_mov_b32_e32 v0, 1
; GCN-NEXT:    s_or_b64 s[6:7], s[6:7], s[12:13]
; GCN-NEXT:    s_branch .LBB10_1
; GCN-NEXT:  .LBB10_4: ; %exit
; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN-NEXT:    s_mov_b32 s32, s34
; GCN-NEXT:    s_mov_b32 s34, s14
; GCN-NEXT:    s_mov_b32 s33, s11
; GCN-NEXT:    s_setpc_b64 s[30:31]
begin:
  %local_var = alloca i32, align 1024, addrspace(5)
  store volatile i32 0, ptr addrspace(5) %local_var, align 1024
  br label %loop_body

loop_end:                                                ; preds = %loop_body
  %idx_next = add nuw nsw i32 %lp_idx, 1
  %lp_exit_cond = icmp eq i32 %idx_next, 9
  br i1 %lp_exit_cond, label %exit, label %loop_body

loop_body:                                                ; preds = %loop_end, %begin
  %lp_idx = phi i32 [ 0, %begin ], [ %idx_next, %loop_end ]
  %ptr = getelementptr inbounds %struct.Data, ptr addrspace(5) %arg, i32 0, i32 0, i32 %lp_idx
  %val = load i32, ptr addrspace(5) %ptr, align 8
  %lp_cond = icmp eq i32 %val, %lp_idx
  br i1 %lp_cond, label %loop_end, label %exit

exit:                                               ; preds = %loop_end, %loop_body
  %out = phi i32 [ 0, %loop_body ], [ 1, %loop_end ]
  ret i32 %out
}

define void @no_free_scratch_sgpr_for_bp_copy(<32 x i32> %a, i32 %b) #0 {
; GCN-LABEL: no_free_scratch_sgpr_for_bp_copy:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT:    s_mov_b32 s41, s34
; GCN-NEXT:    s_mov_b32 s34, s32
; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s34 offset:4
; GCN-NEXT:    s_mov_b32 s40, s33
; GCN-NEXT:    s_add_i32 s33, s32, 0x1fc0
; GCN-NEXT:    s_and_b32 s33, s33, 0xffffe000
; GCN-NEXT:    s_addk_i32 s32, 0x6000
; GCN-NEXT:    s_mov_b32 s32, s34
; GCN-NEXT:    s_mov_b32 s34, s41
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:128
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    ;;#ASMSTART
; GCN-NEXT:    ;;#ASMEND
; GCN-NEXT:    s_mov_b32 s33, s40
; GCN-NEXT:    s_setpc_b64 s[30:31]
  %local_val = alloca i32, align 128, addrspace(5)
  store volatile i32 %b, ptr addrspace(5) %local_val, align 128
  ; Use all clobberable registers, so BP has to spill to a VGPR.
  call void asm sideeffect "",
    "~{s0},~{s1},~{s2},~{s3},~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
    ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
    ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
    ,~{vcc_hi}"() #0
  ret void
}

define void @no_free_regs_spill_bp_to_memory(<32 x i32> %a, i32 %b) #5 {
; If there are no free SGPRs or VGPRs available we must spill the BP to memory.
; GCN-LABEL: no_free_regs_spill_bp_to_memory:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT:    s_mov_b32 s4, s33
; GCN-NEXT:    s_add_i32 s33, s32, 0x1fc0
; GCN-NEXT:    s_and_b32 s33, s33, 0xffffe000
; GCN-NEXT:    s_xor_saveexec_b64 s[6:7], -1
; GCN-NEXT:    buffer_store_dword v39, off, s[0:3], s33 offset:132 ; 4-byte Folded Spill
; GCN-NEXT:    s_mov_b64 exec, s[6:7]
; GCN-NEXT:    v_writelane_b32 v39, s4, 32
; GCN-NEXT:    v_writelane_b32 v39, s34, 33
; GCN-NEXT:    s_mov_b32 s34, s32
; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s34 offset:4
; GCN-NEXT:    v_writelane_b32 v39, s39, 0
; GCN-NEXT:    v_writelane_b32 v39, s48, 1
; GCN-NEXT:    v_writelane_b32 v39, s49, 2
; GCN-NEXT:    v_writelane_b32 v39, s50, 3
; GCN-NEXT:    v_writelane_b32 v39, s51, 4
; GCN-NEXT:    v_writelane_b32 v39, s52, 5
; GCN-NEXT:    v_writelane_b32 v39, s53, 6
; GCN-NEXT:    v_writelane_b32 v39, s54, 7
; GCN-NEXT:    v_writelane_b32 v39, s55, 8
; GCN-NEXT:    v_writelane_b32 v39, s64, 9
; GCN-NEXT:    v_writelane_b32 v39, s65, 10
; GCN-NEXT:    v_writelane_b32 v39, s66, 11
; GCN-NEXT:    v_writelane_b32 v39, s67, 12
; GCN-NEXT:    v_writelane_b32 v39, s68, 13
; GCN-NEXT:    v_writelane_b32 v39, s69, 14
; GCN-NEXT:    v_writelane_b32 v39, s70, 15
; GCN-NEXT:    v_writelane_b32 v39, s71, 16
; GCN-NEXT:    v_writelane_b32 v39, s80, 17
; GCN-NEXT:    v_writelane_b32 v39, s81, 18
; GCN-NEXT:    v_writelane_b32 v39, s82, 19
; GCN-NEXT:    v_writelane_b32 v39, s83, 20
; GCN-NEXT:    v_writelane_b32 v39, s84, 21
; GCN-NEXT:    v_writelane_b32 v39, s85, 22
; GCN-NEXT:    v_writelane_b32 v39, s86, 23
; GCN-NEXT:    v_writelane_b32 v39, s87, 24
; GCN-NEXT:    v_writelane_b32 v39, s96, 25
; GCN-NEXT:    v_writelane_b32 v39, s97, 26
; GCN-NEXT:    v_writelane_b32 v39, s98, 27
; GCN-NEXT:    v_writelane_b32 v39, s99, 28
; GCN-NEXT:    v_writelane_b32 v39, s100, 29
; GCN-NEXT:    v_writelane_b32 v39, s101, 30
; GCN-NEXT:    s_addk_i32 s32, 0x6000
; GCN-NEXT:    v_writelane_b32 v39, s102, 31
; GCN-NEXT:    s_mov_b32 s32, s34
; GCN-NEXT:    v_readlane_b32 s34, v39, 33
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:128
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    ;;#ASMSTART
; GCN-NEXT:    ; clobber nonpreserved SGPRs and 64 CSRs
; GCN-NEXT:    ;;#ASMEND
; GCN-NEXT:    ;;#ASMSTART
; GCN-NEXT:    ; clobber all VGPRs
; GCN-NEXT:    ;;#ASMEND
; GCN-NEXT:    v_readlane_b32 s102, v39, 31
; GCN-NEXT:    v_readlane_b32 s101, v39, 30
; GCN-NEXT:    v_readlane_b32 s100, v39, 29
; GCN-NEXT:    v_readlane_b32 s99, v39, 28
; GCN-NEXT:    v_readlane_b32 s98, v39, 27
; GCN-NEXT:    v_readlane_b32 s97, v39, 26
; GCN-NEXT:    v_readlane_b32 s96, v39, 25
; GCN-NEXT:    v_readlane_b32 s87, v39, 24
; GCN-NEXT:    v_readlane_b32 s86, v39, 23
; GCN-NEXT:    v_readlane_b32 s85, v39, 22
; GCN-NEXT:    v_readlane_b32 s84, v39, 21
; GCN-NEXT:    v_readlane_b32 s83, v39, 20
; GCN-NEXT:    v_readlane_b32 s82, v39, 19
; GCN-NEXT:    v_readlane_b32 s81, v39, 18
; GCN-NEXT:    v_readlane_b32 s80, v39, 17
; GCN-NEXT:    v_readlane_b32 s71, v39, 16
; GCN-NEXT:    v_readlane_b32 s70, v39, 15
; GCN-NEXT:    v_readlane_b32 s69, v39, 14
; GCN-NEXT:    v_readlane_b32 s68, v39, 13
; GCN-NEXT:    v_readlane_b32 s67, v39, 12
; GCN-NEXT:    v_readlane_b32 s66, v39, 11
; GCN-NEXT:    v_readlane_b32 s65, v39, 10
; GCN-NEXT:    v_readlane_b32 s64, v39, 9
; GCN-NEXT:    v_readlane_b32 s55, v39, 8
; GCN-NEXT:    v_readlane_b32 s54, v39, 7
; GCN-NEXT:    v_readlane_b32 s53, v39, 6
; GCN-NEXT:    v_readlane_b32 s52, v39, 5
; GCN-NEXT:    v_readlane_b32 s51, v39, 4
; GCN-NEXT:    v_readlane_b32 s50, v39, 3
; GCN-NEXT:    v_readlane_b32 s49, v39, 2
; GCN-NEXT:    v_readlane_b32 s48, v39, 1
; GCN-NEXT:    v_readlane_b32 s39, v39, 0
; GCN-NEXT:    v_readlane_b32 s4, v39, 32
; GCN-NEXT:    s_xor_saveexec_b64 s[6:7], -1
; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload
; GCN-NEXT:    s_mov_b64 exec, s[6:7]
; GCN-NEXT:    s_mov_b32 s33, s4
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    s_setpc_b64 s[30:31]
  %local_val = alloca i32, align 128, addrspace(5)
  store volatile i32 %b, ptr addrspace(5) %local_val, align 128

  call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
    "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
    ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
    ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
    ,~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49}
    ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59}
    ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69}
    ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79}
    ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89}
    ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99}
    ,~{s100},~{s101},~{s102},~{s39},~{vcc}"() #0

  call void asm sideeffect "; clobber all VGPRs",
    "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
    ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
    ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
    ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38}" () #0
  ret void
}

define void @spill_bp_to_memory_scratch_reg_needed_mubuf_offset(<32 x i32> %a, i32 %b, ptr addrspace(5) byval([4096 x i8]) align 4 %arg) #5 {
; If the size of the offset exceeds the MUBUF offset field we need another
; scratch VGPR to hold the offset.
; GCN-LABEL: spill_bp_to_memory_scratch_reg_needed_mubuf_offset:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT:    s_mov_b32 s4, s33
; GCN-NEXT:    s_add_i32 s33, s32, 0x1fc0
; GCN-NEXT:    s_and_b32 s33, s33, 0xffffe000
; GCN-NEXT:    s_xor_saveexec_b64 s[6:7], -1
; GCN-NEXT:    s_add_i32 s5, s33, 0x42100
; GCN-NEXT:    buffer_store_dword v39, off, s[0:3], s5 ; 4-byte Folded Spill
; GCN-NEXT:    s_mov_b64 exec, s[6:7]
; GCN-NEXT:    v_writelane_b32 v39, s4, 32
; GCN-NEXT:    v_writelane_b32 v39, s34, 33
; GCN-NEXT:    s_mov_b32 s34, s32
; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s34 offset:4
; GCN-NEXT:    v_writelane_b32 v39, s39, 0
; GCN-NEXT:    v_writelane_b32 v39, s48, 1
; GCN-NEXT:    v_writelane_b32 v39, s49, 2
; GCN-NEXT:    v_writelane_b32 v39, s50, 3
; GCN-NEXT:    v_writelane_b32 v39, s51, 4
; GCN-NEXT:    v_writelane_b32 v39, s52, 5
; GCN-NEXT:    v_writelane_b32 v39, s53, 6
; GCN-NEXT:    v_writelane_b32 v39, s54, 7
; GCN-NEXT:    v_writelane_b32 v39, s55, 8
; GCN-NEXT:    v_writelane_b32 v39, s64, 9
; GCN-NEXT:    v_writelane_b32 v39, s65, 10
; GCN-NEXT:    v_writelane_b32 v39, s66, 11
; GCN-NEXT:    v_writelane_b32 v39, s67, 12
; GCN-NEXT:    v_writelane_b32 v39, s68, 13
; GCN-NEXT:    v_writelane_b32 v39, s69, 14
; GCN-NEXT:    v_writelane_b32 v39, s70, 15
; GCN-NEXT:    v_writelane_b32 v39, s71, 16
; GCN-NEXT:    v_writelane_b32 v39, s80, 17
; GCN-NEXT:    v_writelane_b32 v39, s81, 18
; GCN-NEXT:    v_writelane_b32 v39, s82, 19
; GCN-NEXT:    v_writelane_b32 v39, s83, 20
; GCN-NEXT:    v_writelane_b32 v39, s84, 21
; GCN-NEXT:    v_writelane_b32 v39, s85, 22
; GCN-NEXT:    v_writelane_b32 v39, s86, 23
; GCN-NEXT:    v_writelane_b32 v39, s87, 24
; GCN-NEXT:    v_writelane_b32 v39, s96, 25
; GCN-NEXT:    v_writelane_b32 v39, s97, 26
; GCN-NEXT:    v_writelane_b32 v39, s98, 27
; GCN-NEXT:    v_writelane_b32 v39, s99, 28
; GCN-NEXT:    v_writelane_b32 v39, s100, 29
; GCN-NEXT:    v_writelane_b32 v39, s101, 30
; GCN-NEXT:    v_mov_b32_e32 v1, 0x1080
; GCN-NEXT:    s_add_i32 s32, s32, 0x46000
; GCN-NEXT:    v_writelane_b32 v39, s102, 31
; GCN-NEXT:    s_mov_b32 s32, s34
; GCN-NEXT:    v_readlane_b32 s34, v39, 33
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    buffer_store_dword v0, v1, s[0:3], s33 offen
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    ;;#ASMSTART
; GCN-NEXT:    ; clobber nonpreserved SGPRs and 64 CSRs
; GCN-NEXT:    ;;#ASMEND
; GCN-NEXT:    ;;#ASMSTART
; GCN-NEXT:    ; clobber all VGPRs
; GCN-NEXT:    ;;#ASMEND
; GCN-NEXT:    v_readlane_b32 s102, v39, 31
; GCN-NEXT:    v_readlane_b32 s101, v39, 30
; GCN-NEXT:    v_readlane_b32 s100, v39, 29
; GCN-NEXT:    v_readlane_b32 s99, v39, 28
; GCN-NEXT:    v_readlane_b32 s98, v39, 27
; GCN-NEXT:    v_readlane_b32 s97, v39, 26
; GCN-NEXT:    v_readlane_b32 s96, v39, 25
; GCN-NEXT:    v_readlane_b32 s87, v39, 24
; GCN-NEXT:    v_readlane_b32 s86, v39, 23
; GCN-NEXT:    v_readlane_b32 s85, v39, 22
; GCN-NEXT:    v_readlane_b32 s84, v39, 21
; GCN-NEXT:    v_readlane_b32 s83, v39, 20
; GCN-NEXT:    v_readlane_b32 s82, v39, 19
; GCN-NEXT:    v_readlane_b32 s81, v39, 18
; GCN-NEXT:    v_readlane_b32 s80, v39, 17
; GCN-NEXT:    v_readlane_b32 s71, v39, 16
; GCN-NEXT:    v_readlane_b32 s70, v39, 15
; GCN-NEXT:    v_readlane_b32 s69, v39, 14
; GCN-NEXT:    v_readlane_b32 s68, v39, 13
; GCN-NEXT:    v_readlane_b32 s67, v39, 12
; GCN-NEXT:    v_readlane_b32 s66, v39, 11
; GCN-NEXT:    v_readlane_b32 s65, v39, 10
; GCN-NEXT:    v_readlane_b32 s64, v39, 9
; GCN-NEXT:    v_readlane_b32 s55, v39, 8
; GCN-NEXT:    v_readlane_b32 s54, v39, 7
; GCN-NEXT:    v_readlane_b32 s53, v39, 6
; GCN-NEXT:    v_readlane_b32 s52, v39, 5
; GCN-NEXT:    v_readlane_b32 s51, v39, 4
; GCN-NEXT:    v_readlane_b32 s50, v39, 3
; GCN-NEXT:    v_readlane_b32 s49, v39, 2
; GCN-NEXT:    v_readlane_b32 s48, v39, 1
; GCN-NEXT:    v_readlane_b32 s39, v39, 0
; GCN-NEXT:    v_readlane_b32 s4, v39, 32
; GCN-NEXT:    s_xor_saveexec_b64 s[6:7], -1
; GCN-NEXT:    s_add_i32 s5, s33, 0x42100
; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s5 ; 4-byte Folded Reload
; GCN-NEXT:    s_mov_b64 exec, s[6:7]
; GCN-NEXT:    s_mov_b32 s33, s4
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    s_setpc_b64 s[30:31]
  %local_val = alloca i32, align 128, addrspace(5)
  store volatile i32 %b, ptr addrspace(5) %local_val, align 128

  call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
    "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
    ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
    ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
    ,~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49}
    ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59}
    ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69}
    ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79}
    ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89}
    ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99}
    ,~{s100},~{s101},~{s102},~{s39},~{vcc}"() #0

  call void asm sideeffect "; clobber all VGPRs",
    "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
    ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
    ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
    ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38}"() #0
  ret void
}

attributes #0 = { noinline nounwind }
attributes #1 = { noinline nounwind "stackrealign" }
attributes #2 = { noinline nounwind alignstack=4 }
attributes #3 = { noinline nounwind "no-realign-stack" }
attributes #4 = { noinline nounwind "frame-pointer"="all"}
attributes #5 = { noinline nounwind "amdgpu-waves-per-eu"="6,6" }