; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -O3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck %s

%"struct.__llvm_libc::rpc::Buffer" = type { [8 x i64] }

define void @issue63986(i64 %0, i64 %idxprom, ptr inreg %ptr) {
; CHECK-LABEL: issue63986:
; CHECK:       ; %bb.0: ; %entry
; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT:    v_lshlrev_b64 v[4:5], 6, v[2:3]
; CHECK-NEXT:    v_mov_b32_e32 v6, s17
; CHECK-NEXT:    v_add_co_u32_e32 v8, vcc, s16, v4
; CHECK-NEXT:    v_addc_co_u32_e32 v9, vcc, v6, v5, vcc
; CHECK-NEXT:    s_mov_b64 s[4:5], 0
; CHECK-NEXT:  .LBB0_1: ; %loop-memcpy-expansion
; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    v_mov_b32_e32 v7, s5
; CHECK-NEXT:    v_mov_b32_e32 v6, s4
; CHECK-NEXT:    flat_load_dwordx4 v[10:13], v[6:7]
; CHECK-NEXT:    v_add_co_u32_e32 v6, vcc, s4, v8
; CHECK-NEXT:    s_add_u32 s4, s4, 16
; CHECK-NEXT:    s_addc_u32 s5, s5, 0
; CHECK-NEXT:    v_cmp_ge_u64_e64 s[6:7], s[4:5], 32
; CHECK-NEXT:    v_addc_co_u32_e32 v7, vcc, v9, v7, vcc
; CHECK-NEXT:    s_and_b64 vcc, exec, s[6:7]
; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT:    flat_store_dwordx4 v[6:7], v[10:13]
; CHECK-NEXT:    s_cbranch_vccz .LBB0_1
; CHECK-NEXT:  ; %bb.2: ; %loop-memcpy-residual-header
; CHECK-NEXT:    s_branch .LBB0_4
; CHECK-NEXT:  ; %bb.3:
; CHECK-NEXT:    ; implicit-def: $vgpr6_vgpr7
; CHECK-NEXT:    s_branch .LBB0_5
; CHECK-NEXT:  .LBB0_4: ; %loop-memcpy-residual-header.post-loop-memcpy-expansion_crit_edge
; CHECK-NEXT:    v_lshlrev_b64 v[6:7], 6, v[2:3]
; CHECK-NEXT:    s_cbranch_execnz .LBB0_8
; CHECK-NEXT:  .LBB0_5: ; %loop-memcpy-residual.preheader
; CHECK-NEXT:    s_add_u32 s4, s16, 32
; CHECK-NEXT:    s_addc_u32 s5, s17, 0
; CHECK-NEXT:    v_mov_b32_e32 v3, s5
; CHECK-NEXT:    v_add_co_u32_e32 v2, vcc, s4, v4
; CHECK-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v5, vcc
; CHECK-NEXT:    s_mov_b64 s[4:5], 0
; CHECK-NEXT:  ; %bb.6: ; %loop-memcpy-residual
; CHECK-NEXT:    s_add_u32 s6, 32, s4
; CHECK-NEXT:    s_addc_u32 s7, 0, s5
; CHECK-NEXT:    v_mov_b32_e32 v6, s6
; CHECK-NEXT:    v_mov_b32_e32 v7, s7
; CHECK-NEXT:    flat_load_ubyte v10, v[6:7]
; CHECK-NEXT:    v_mov_b32_e32 v7, s5
; CHECK-NEXT:    v_add_co_u32_e32 v6, vcc, s4, v2
; CHECK-NEXT:    v_addc_co_u32_e32 v7, vcc, v3, v7, vcc
; CHECK-NEXT:    s_add_u32 s4, s4, 1
; CHECK-NEXT:    s_addc_u32 s5, s5, 0
; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT:    flat_store_byte v[6:7], v10
; CHECK-NEXT:  ; %bb.7:
; CHECK-NEXT:    v_mov_b32_e32 v7, v5
; CHECK-NEXT:    v_mov_b32_e32 v6, v4
; CHECK-NEXT:  .LBB0_8: ; %post-loop-memcpy-expansion
; CHECK-NEXT:    v_and_b32_e32 v2, 15, v0
; CHECK-NEXT:    v_and_b32_e32 v0, -16, v0
; CHECK-NEXT:    v_add_co_u32_e32 v4, vcc, v6, v0
; CHECK-NEXT:    v_mov_b32_e32 v3, 0
; CHECK-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v1, vcc
; CHECK-NEXT:    v_cmp_ne_u64_e64 s[4:5], 0, v[0:1]
; CHECK-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[2:3]
; CHECK-NEXT:    v_mov_b32_e32 v6, s17
; CHECK-NEXT:    v_add_co_u32_e32 v4, vcc, s16, v4
; CHECK-NEXT:    v_addc_co_u32_e32 v5, vcc, v6, v5, vcc
; CHECK-NEXT:    s_branch .LBB0_11
; CHECK-NEXT:  .LBB0_9: ; %Flow14
; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
; CHECK-NEXT:    s_or_b64 exec, exec, s[10:11]
; CHECK-NEXT:    s_mov_b64 s[8:9], 0
; CHECK-NEXT:  .LBB0_10: ; %Flow16
; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
; CHECK-NEXT:    s_cbranch_vccz .LBB0_19
; CHECK-NEXT:  .LBB0_11: ; %while.cond
; CHECK-NEXT:    ; =>This Loop Header: Depth=1
; CHECK-NEXT:    ; Child Loop BB0_13 Depth 2
; CHECK-NEXT:    ; Child Loop BB0_17 Depth 2
; CHECK-NEXT:    s_and_saveexec_b64 s[8:9], s[4:5]
; CHECK-NEXT:    s_cbranch_execz .LBB0_14
; CHECK-NEXT:  ; %bb.12: ; %loop-memcpy-expansion2.preheader
; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
; CHECK-NEXT:    s_mov_b64 s[10:11], 0
; CHECK-NEXT:    s_mov_b64 s[12:13], 0
; CHECK-NEXT:  .LBB0_13: ; %loop-memcpy-expansion2
; CHECK-NEXT:    ; Parent Loop BB0_11 Depth=1
; CHECK-NEXT:    ; => This Inner Loop Header: Depth=2
; CHECK-NEXT:    v_mov_b32_e32 v6, s12
; CHECK-NEXT:    v_mov_b32_e32 v7, s13
; CHECK-NEXT:    flat_load_dwordx4 v[10:13], v[6:7]
; CHECK-NEXT:    v_add_co_u32_e32 v6, vcc, s12, v8
; CHECK-NEXT:    s_add_u32 s12, s12, 16
; CHECK-NEXT:    v_addc_co_u32_e32 v7, vcc, v9, v7, vcc
; CHECK-NEXT:    s_addc_u32 s13, s13, 0
; CHECK-NEXT:    v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]
; CHECK-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT:    flat_store_dwordx4 v[6:7], v[10:13]
; CHECK-NEXT:    s_andn2_b64 exec, exec, s[10:11]
; CHECK-NEXT:    s_cbranch_execnz .LBB0_13
; CHECK-NEXT:  .LBB0_14: ; %Flow15
; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
; CHECK-NEXT:    s_or_b64 exec, exec, s[8:9]
; CHECK-NEXT:    s_mov_b64 s[8:9], -1
; CHECK-NEXT:    s_cbranch_execz .LBB0_10
; CHECK-NEXT:  ; %bb.15: ; %loop-memcpy-residual-header5
; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
; CHECK-NEXT:    s_and_saveexec_b64 s[8:9], s[6:7]
; CHECK-NEXT:    s_xor_b64 s[10:11], exec, s[8:9]
; CHECK-NEXT:    s_cbranch_execz .LBB0_9
; CHECK-NEXT:  ; %bb.16: ; %loop-memcpy-residual4.preheader
; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
; CHECK-NEXT:    s_mov_b64 s[12:13], 0
; CHECK-NEXT:    s_mov_b64 s[14:15], 0
; CHECK-NEXT:  .LBB0_17: ; %loop-memcpy-residual4
; CHECK-NEXT:    ; Parent Loop BB0_11 Depth=1
; CHECK-NEXT:    ; => This Inner Loop Header: Depth=2
; CHECK-NEXT:    v_mov_b32_e32 v10, s15
; CHECK-NEXT:    v_add_co_u32_e32 v6, vcc, s14, v0
; CHECK-NEXT:    v_addc_co_u32_e32 v7, vcc, v1, v10, vcc
; CHECK-NEXT:    flat_load_ubyte v11, v[6:7]
; CHECK-NEXT:    v_add_co_u32_e32 v6, vcc, s14, v4
; CHECK-NEXT:    s_add_u32 s14, s14, 1
; CHECK-NEXT:    s_addc_u32 s15, s15, 0
; CHECK-NEXT:    v_cmp_ge_u64_e64 s[8:9], s[14:15], v[2:3]
; CHECK-NEXT:    v_addc_co_u32_e32 v7, vcc, v5, v10, vcc
; CHECK-NEXT:    s_or_b64 s[12:13], s[8:9], s[12:13]
; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT:    flat_store_byte v[6:7], v11
; CHECK-NEXT:    s_andn2_b64 exec, exec, s[12:13]
; CHECK-NEXT:    s_cbranch_execnz .LBB0_17
; CHECK-NEXT:  ; %bb.18: ; %Flow
; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
; CHECK-NEXT:    s_or_b64 exec, exec, s[12:13]
; CHECK-NEXT:    s_branch .LBB0_9
; CHECK-NEXT:  .LBB0_19: ; %DummyReturnBlock
; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT:    s_setpc_b64 s[30:31]
entry:
  %arrayidx = getelementptr [32 x %"struct.__llvm_libc::rpc::Buffer"], ptr %ptr, i64 0, i64 %idxprom
  %spec.select = tail call i64 @llvm.umin.i64(i64 sub (i64 ptrtoint (ptr addrspacecast (ptr addrspace(4) inttoptr (i64 32 to ptr addrspace(4)) to ptr) to i64), i64 ptrtoint (ptr addrspacecast (ptr addrspace(4) null to ptr) to i64)), i64 56)
  tail call void @llvm.memcpy.p0.p0.i64(ptr %arrayidx, ptr null, i64 %spec.select, i1 false)
  br label %while.cond

while.cond:                         ; preds = %while.cond
  tail call void @llvm.memcpy.p0.p0.i64(ptr %arrayidx, ptr null, i64 %0, i1 false)
  br label %while.cond
}

define void @issue63986_reduced_expanded(i64 %idxprom) {
; CHECK-LABEL: issue63986_reduced_expanded:
; CHECK:       ; %bb.0: ; %entry
; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT:  ; %bb.1: ; %loop-memcpy-expansion.preheader
; CHECK-NEXT:    s_setpc_b64 s[30:31]
; CHECK-NEXT:  ; %bb.2: ; %loop-memcpy-residual-header
; CHECK-NEXT:    s_and_b32 s4, 32, 15
; CHECK-NEXT:    s_mov_b32 s5, 0
; CHECK-NEXT:    s_cbranch_scc0 .LBB1_4
; CHECK-NEXT:  ; %bb.3:
; CHECK-NEXT:    ; implicit-def: $vgpr0_vgpr1
; CHECK-NEXT:    s_branch .LBB1_5
; CHECK-NEXT:  .LBB1_4: ; %loop-memcpy-residual-header.post-loop-memcpy-expansion_crit_edge
; CHECK-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
; CHECK-NEXT:    s_cbranch_execnz .LBB1_8
; CHECK-NEXT:  .LBB1_5: ; %loop-memcpy-residual.preheader
; CHECK-NEXT:    v_mov_b32_e32 v0, s4
; CHECK-NEXT:    s_mov_b64 s[8:9], 0
; CHECK-NEXT:    s_mov_b32 s7, 0
; CHECK-NEXT:    v_mov_b32_e32 v1, s5
; CHECK-NEXT:  .LBB1_6: ; %loop-memcpy-residual
; CHECK-NEXT:    s_add_i32 s6, s8, 1
; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
; CHECK-NEXT:    s_mov_b64 s[8:9], 1
; CHECK-NEXT:    s_cbranch_vccnz .LBB1_6
; CHECK-NEXT:  ; %bb.7: ; %Flow
; CHECK-NEXT:    v_mov_b32_e32 v0, 0
; CHECK-NEXT:    v_mov_b32_e32 v1, 0
; CHECK-NEXT:  .LBB1_8: ; %post-loop-memcpy-expansion
; CHECK-NEXT:    v_mov_b32_e32 v2, 0
; CHECK-NEXT:    v_mov_b32_e32 v3, v2
; CHECK-NEXT:    v_mov_b32_e32 v4, v2
; CHECK-NEXT:    v_mov_b32_e32 v5, v2
; CHECK-NEXT:    s_and_b64 vcc, exec, 0
; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
; CHECK-NEXT:  .LBB1_9: ; %loop-memcpy-expansion2
; CHECK-NEXT:    s_mov_b64 vcc, vcc
; CHECK-NEXT:    s_cbranch_vccz .LBB1_9
; CHECK-NEXT:  ; %bb.10: ; %DummyReturnBlock
; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT:    s_setpc_b64 s[30:31]
entry:
  %spec.select = tail call i64 @llvm.umin.i64(i64 sub (i64 ptrtoint (ptr addrspacecast (ptr addrspace(4) inttoptr (i64 32 to ptr addrspace(4)) to ptr) to i64), i64 ptrtoint (ptr addrspacecast (ptr addrspace(4) null to ptr) to i64)), i64 56)
  %i = trunc i64 %spec.select to i32
  %i1 = urem i32 %i, 16
  %i2 = zext i32 %i to i64
  %i3 = zext i32 %i1 to i64
  %i4 = icmp ne i64 %i2, 0
  br i1 %i4, label %loop-memcpy-expansion.preheader, label %loop-memcpy-residual-header

loop-memcpy-expansion.preheader:                  ; preds = %entry
  ret void

loop-memcpy-residual:                             ; preds = %loop-memcpy-residual.preheader, %loop-memcpy-residual
  %residual-loop-index1 = phi i64 [ 1, %loop-memcpy-residual ], [ 0, %loop-memcpy-residual.preheader ]
  %i5 = add i64 %residual-loop-index1, 1
  %i6 = icmp ult i64 %i5, %i3
  br i1 %i6, label %loop-memcpy-residual, label %post-loop-memcpy-expansion

post-loop-memcpy-expansion:                       ; preds = %loop-memcpy-residual-header.post-loop-memcpy-expansion_crit_edge, %loop-memcpy-residual
  %.pre-phi = phi i64 [ %.pre, %loop-memcpy-residual-header.post-loop-memcpy-expansion_crit_edge ], [ 0, %loop-memcpy-residual ]
  br label %loop-memcpy-expansion2

loop-memcpy-expansion2:                           ; preds = %loop-memcpy-expansion2, %post-loop-memcpy-expansion
  %scevgep7 = getelementptr i8, ptr null, i64 %.pre-phi
  store <4 x i32> zeroinitializer, ptr %scevgep7, align 1
  br label %loop-memcpy-expansion2

loop-memcpy-residual-header:                      ; preds = %entry
  %i7 = icmp ne i64 %i3, 0
  br i1 %i7, label %loop-memcpy-residual.preheader, label %loop-memcpy-residual-header.post-loop-memcpy-expansion_crit_edge

loop-memcpy-residual-header.post-loop-memcpy-expansion_crit_edge: ; preds = %loop-memcpy-residual-header
  %.pre = shl i64 %idxprom, 1
  br label %post-loop-memcpy-expansion

loop-memcpy-residual.preheader:                   ; preds = %loop-memcpy-residual-header
  br label %loop-memcpy-residual
}

declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #0
declare i64 @llvm.umin.i64(i64, i64) #1

attributes #0 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }