; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=GCN1 %s ; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GCN2 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN3 %s ; --------------------------------------------------------------------- ; atomicrmw xchg ; --------------------------------------------------------------------- define void @flat_atomic_xchg_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_xchg_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB0_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB0_4 ; GCN1-NEXT: .LBB0_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB0_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB0_2 ; GCN1-NEXT: .LBB0_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0 ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB0_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB0_4 ; GCN2-NEXT: .LBB0_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB0_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB0_2 ; GCN2-NEXT: .LBB0_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0 ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB0_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB0_4 ; GCN3-NEXT: .LBB0_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB0_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB0_2 ; GCN3-NEXT: .LBB0_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_xchg_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB1_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB1_4 ; GCN1-NEXT: .LBB1_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB1_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB1_2 ; GCN1-NEXT: .LBB1_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0 ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB1_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB1_4 ; GCN2-NEXT: .LBB1_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB1_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB1_2 ; GCN2-NEXT: .LBB1_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0 ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB1_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB1_4 ; GCN3-NEXT: .LBB1_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB1_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB1_2 ; GCN3-NEXT: .LBB1_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_xchg_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_xchg_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_mov_b32_e32 v5, v1 ; GCN1-NEXT: v_mov_b32_e32 v4, v0 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB2_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB2_4 ; GCN1-NEXT: .LBB2_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB2_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB2_2 ; GCN1-NEXT: .LBB2_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_mov_b32_e32 v5, v1 ; GCN2-NEXT: v_mov_b32_e32 v4, v0 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB2_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB2_4 ; GCN2-NEXT: .LBB2_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB2_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB2_2 ; GCN2-NEXT: .LBB2_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB2_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB2_4 ; GCN3-NEXT: .LBB2_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB2_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB2_2 ; GCN3-NEXT: .LBB2_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_nop 0 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw xchg ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_xchg_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB3_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB3_4 ; GCN1-NEXT: .LBB3_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB3_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB3_2 ; GCN1-NEXT: .LBB3_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB3_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB3_4 ; GCN2-NEXT: .LBB3_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB3_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB3_2 ; GCN2-NEXT: .LBB3_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB3_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB3_4 ; GCN3-NEXT: .LBB3_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB3_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB3_2 ; GCN3-NEXT: .LBB3_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_nop 0 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw xchg ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_xchg_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_xchg_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB4_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB4_4 ; GCN1-NEXT: .LBB4_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB4_3: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v0, s4 ; GCN1-NEXT: v_mov_b32_e32 v2, s6 ; GCN1-NEXT: v_mov_b32_e32 v1, s5 ; GCN1-NEXT: v_mov_b32_e32 v3, s7 ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: s_cbranch_execnz .LBB4_2 ; GCN1-NEXT: .LBB4_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v0, s7 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB4_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB4_4 ; GCN2-NEXT: .LBB4_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB4_3: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v0, s4 ; GCN2-NEXT: v_mov_b32_e32 v2, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s5 ; GCN2-NEXT: v_mov_b32_e32 v3, s7 ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: s_cbranch_execnz .LBB4_2 ; GCN2-NEXT: .LBB4_4: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v0, s7 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB4_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB4_4 ; GCN3-NEXT: .LBB4_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB4_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v0, s4 ; GCN3-NEXT: v_mov_b32_e32 v2, s6 ; GCN3-NEXT: v_mov_b32_e32 v1, s5 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: s_cbranch_execnz .LBB4_2 ; GCN3-NEXT: .LBB4_4: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s7 ; GCN3-NEXT: v_mov_b32_e32 v1, s34 ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v0, s6 ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_xchg_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB5_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB5_4 ; GCN1-NEXT: .LBB5_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB5_3: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v2, s6 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v3, s7 ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: s_cbranch_execnz .LBB5_2 ; GCN1-NEXT: .LBB5_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s6 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v0, s7 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB5_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB5_4 ; GCN2-NEXT: .LBB5_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB5_3: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v2, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v3, s7 ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: s_cbranch_execnz .LBB5_2 ; GCN2-NEXT: .LBB5_4: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v0, s7 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB5_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB5_4 ; GCN3-NEXT: .LBB5_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB5_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: v_mov_b32_e32 v2, s6 ; GCN3-NEXT: v_mov_b32_e32 v1, s35 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: s_cbranch_execnz .LBB5_2 ; GCN3-NEXT: .LBB5_4: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s7 ; GCN3-NEXT: v_mov_b32_e32 v1, s34 ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v0, s6 ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_xchg_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB6_2 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v0, s4 ; GCN1-NEXT: v_mov_b32_e32 v2, s6 ; GCN1-NEXT: v_mov_b32_e32 v1, s5 ; GCN1-NEXT: v_mov_b32_e32 v3, s7 ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: s_cbranch_execz .LBB6_3 ; GCN1-NEXT: s_branch .LBB6_4 ; GCN1-NEXT: .LBB6_2: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: .LBB6_3: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v2, s7 ; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB6_4: ; %atomicrmw.end ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB6_2 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v0, s4 ; GCN2-NEXT: v_mov_b32_e32 v2, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s5 ; GCN2-NEXT: v_mov_b32_e32 v3, s7 ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: s_cbranch_execz .LBB6_3 ; GCN2-NEXT: s_branch .LBB6_4 ; GCN2-NEXT: .LBB6_2: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: .LBB6_3: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v4, s6 ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v2, s7 ; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB6_4: ; %atomicrmw.end ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB6_2 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v0, s4 ; GCN3-NEXT: v_mov_b32_e32 v2, s6 ; GCN3-NEXT: v_mov_b32_e32 v1, s5 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: s_cbranch_execz .LBB6_3 ; GCN3-NEXT: s_branch .LBB6_4 ; GCN3-NEXT: .LBB6_2: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: .LBB6_3: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v3, s6 ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB6_4: ; %atomicrmw.end ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw xchg ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB7_2 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v2, s6 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v3, s7 ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: s_cbranch_execz .LBB7_3 ; GCN1-NEXT: s_branch .LBB7_4 ; GCN1-NEXT: .LBB7_2: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: .LBB7_3: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s6 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v2, s7 ; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB7_4: ; %atomicrmw.end ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB7_2 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v2, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v3, s7 ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: s_cbranch_execz .LBB7_3 ; GCN2-NEXT: s_branch .LBB7_4 ; GCN2-NEXT: .LBB7_2: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: .LBB7_3: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v4, s6 ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v2, s7 ; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB7_4: ; %atomicrmw.end ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB7_2 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: v_mov_b32_e32 v2, s6 ; GCN3-NEXT: v_mov_b32_e32 v1, s35 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: s_cbranch_execz .LBB7_3 ; GCN3-NEXT: s_branch .LBB7_4 ; GCN3-NEXT: .LBB7_2: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: .LBB7_3: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v3, s6 ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB7_4: ; %atomicrmw.end ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw xchg ptr %gep, i64 %in seq_cst ret i64 %result } define void @flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB8_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB8_4 ; GCN1-NEXT: .LBB8_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB8_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB8_2 ; GCN1-NEXT: .LBB8_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0 ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB8_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB8_4 ; GCN2-NEXT: .LBB8_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB8_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB8_2 ; GCN2-NEXT: .LBB8_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0 ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB8_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB8_4 ; GCN3-NEXT: .LBB8_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB8_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB8_2 ; GCN3-NEXT: .LBB8_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB9_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB9_4 ; GCN1-NEXT: .LBB9_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB9_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB9_2 ; GCN1-NEXT: .LBB9_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB9_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB9_4 ; GCN2-NEXT: .LBB9_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB9_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB9_2 ; GCN2-NEXT: .LBB9_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB9_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB9_4 ; GCN3-NEXT: .LBB9_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB9_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB9_2 ; GCN3-NEXT: .LBB9_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_nop 0 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } ; --------------------------------------------------------------------- ; atomicrmw xchg f64 ; --------------------------------------------------------------------- define void @flat_atomic_xchg_f64_noret(ptr %ptr, double %in) { ; GCN1-LABEL: flat_atomic_xchg_f64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB10_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB10_4 ; GCN1-NEXT: .LBB10_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB10_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB10_2 ; GCN1-NEXT: .LBB10_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0 ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_f64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB10_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB10_4 ; GCN2-NEXT: .LBB10_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB10_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB10_2 ; GCN2-NEXT: .LBB10_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0 ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_f64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB10_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB10_4 ; GCN3-NEXT: .LBB10_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB10_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB10_2 ; GCN3-NEXT: .LBB10_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst ret void } define void @flat_atomic_xchg_f64_noret_offset(ptr %out, double %in) { ; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB11_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB11_4 ; GCN1-NEXT: .LBB11_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB11_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB11_2 ; GCN1-NEXT: .LBB11_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0 ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB11_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB11_4 ; GCN2-NEXT: .LBB11_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB11_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB11_2 ; GCN2-NEXT: .LBB11_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0 ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB11_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB11_4 ; GCN3-NEXT: .LBB11_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB11_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB11_2 ; GCN3-NEXT: .LBB11_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr double, ptr %out, i32 4 %tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst ret void } define double @flat_atomic_xchg_f64_ret(ptr %ptr, double %in) { ; GCN1-LABEL: flat_atomic_xchg_f64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_mov_b32_e32 v5, v1 ; GCN1-NEXT: v_mov_b32_e32 v4, v0 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB12_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB12_4 ; GCN1-NEXT: .LBB12_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB12_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB12_2 ; GCN1-NEXT: .LBB12_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_f64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_mov_b32_e32 v5, v1 ; GCN2-NEXT: v_mov_b32_e32 v4, v0 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB12_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB12_4 ; GCN2-NEXT: .LBB12_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB12_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB12_2 ; GCN2-NEXT: .LBB12_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_f64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB12_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB12_4 ; GCN3-NEXT: .LBB12_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB12_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB12_2 ; GCN3-NEXT: .LBB12_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_nop 0 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw xchg ptr %ptr, double %in seq_cst ret double %result } define double @flat_atomic_xchg_f64_ret_offset(ptr %out, double %in) { ; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB13_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB13_4 ; GCN1-NEXT: .LBB13_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB13_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB13_2 ; GCN1-NEXT: .LBB13_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB13_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB13_4 ; GCN2-NEXT: .LBB13_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB13_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB13_2 ; GCN2-NEXT: .LBB13_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB13_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB13_4 ; GCN3-NEXT: .LBB13_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB13_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB13_2 ; GCN3-NEXT: .LBB13_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_nop 0 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr double, ptr %out, i32 4 %result = atomicrmw xchg ptr %gep, double %in seq_cst ret double %result } define amdgpu_gfx void @flat_atomic_xchg_f64_noret_scalar(ptr inreg %ptr, double inreg %in) { ; GCN1-LABEL: flat_atomic_xchg_f64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB14_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB14_4 ; GCN1-NEXT: .LBB14_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB14_3: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v0, s4 ; GCN1-NEXT: v_mov_b32_e32 v2, s6 ; GCN1-NEXT: v_mov_b32_e32 v1, s5 ; GCN1-NEXT: v_mov_b32_e32 v3, s7 ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: s_cbranch_execnz .LBB14_2 ; GCN1-NEXT: .LBB14_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v0, s7 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_f64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB14_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB14_4 ; GCN2-NEXT: .LBB14_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB14_3: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v0, s4 ; GCN2-NEXT: v_mov_b32_e32 v2, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s5 ; GCN2-NEXT: v_mov_b32_e32 v3, s7 ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: s_cbranch_execnz .LBB14_2 ; GCN2-NEXT: .LBB14_4: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v0, s7 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_f64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB14_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB14_4 ; GCN3-NEXT: .LBB14_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB14_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v0, s4 ; GCN3-NEXT: v_mov_b32_e32 v2, s6 ; GCN3-NEXT: v_mov_b32_e32 v1, s5 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: s_cbranch_execnz .LBB14_2 ; GCN3-NEXT: .LBB14_4: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s7 ; GCN3-NEXT: v_mov_b32_e32 v1, s34 ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v0, s6 ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_xchg_f64_noret_offset_scalar(ptr inreg %out, double inreg %in) { ; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB15_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB15_4 ; GCN1-NEXT: .LBB15_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB15_3: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v2, s6 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v3, s7 ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: s_cbranch_execnz .LBB15_2 ; GCN1-NEXT: .LBB15_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s6 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v0, s7 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB15_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB15_4 ; GCN2-NEXT: .LBB15_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB15_3: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v2, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v3, s7 ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: s_cbranch_execnz .LBB15_2 ; GCN2-NEXT: .LBB15_4: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v0, s7 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB15_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB15_4 ; GCN3-NEXT: .LBB15_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB15_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: v_mov_b32_e32 v2, s6 ; GCN3-NEXT: v_mov_b32_e32 v1, s35 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: s_cbranch_execnz .LBB15_2 ; GCN3-NEXT: .LBB15_4: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s7 ; GCN3-NEXT: v_mov_b32_e32 v1, s34 ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v0, s6 ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr double, ptr %out, i32 4 %tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst ret void } define amdgpu_gfx double @flat_atomic_xchg_f64_ret_scalar(ptr inreg %ptr, double inreg %in) { ; GCN1-LABEL: flat_atomic_xchg_f64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB16_2 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v0, s4 ; GCN1-NEXT: v_mov_b32_e32 v2, s6 ; GCN1-NEXT: v_mov_b32_e32 v1, s5 ; GCN1-NEXT: v_mov_b32_e32 v3, s7 ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: s_cbranch_execz .LBB16_3 ; GCN1-NEXT: s_branch .LBB16_4 ; GCN1-NEXT: .LBB16_2: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: .LBB16_3: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v2, s7 ; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB16_4: ; %atomicrmw.end ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_f64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB16_2 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v0, s4 ; GCN2-NEXT: v_mov_b32_e32 v2, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s5 ; GCN2-NEXT: v_mov_b32_e32 v3, s7 ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: s_cbranch_execz .LBB16_3 ; GCN2-NEXT: s_branch .LBB16_4 ; GCN2-NEXT: .LBB16_2: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: .LBB16_3: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v4, s6 ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v2, s7 ; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB16_4: ; %atomicrmw.end ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_f64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB16_2 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v0, s4 ; GCN3-NEXT: v_mov_b32_e32 v2, s6 ; GCN3-NEXT: v_mov_b32_e32 v1, s5 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: s_cbranch_execz .LBB16_3 ; GCN3-NEXT: s_branch .LBB16_4 ; GCN3-NEXT: .LBB16_2: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: .LBB16_3: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v3, s6 ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB16_4: ; %atomicrmw.end ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw xchg ptr %ptr, double %in seq_cst ret double %result } define amdgpu_gfx double @flat_atomic_xchg_f64_ret_offset_scalar(ptr inreg %out, double inreg %in) { ; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB17_2 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v2, s6 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v3, s7 ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: s_cbranch_execz .LBB17_3 ; GCN1-NEXT: s_branch .LBB17_4 ; GCN1-NEXT: .LBB17_2: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: .LBB17_3: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s6 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v2, s7 ; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB17_4: ; %atomicrmw.end ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB17_2 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v2, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v3, s7 ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: s_cbranch_execz .LBB17_3 ; GCN2-NEXT: s_branch .LBB17_4 ; GCN2-NEXT: .LBB17_2: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: .LBB17_3: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v4, s6 ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v2, s7 ; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB17_4: ; %atomicrmw.end ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB17_2 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: v_mov_b32_e32 v2, s6 ; GCN3-NEXT: v_mov_b32_e32 v1, s35 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: s_cbranch_execz .LBB17_3 ; GCN3-NEXT: s_branch .LBB17_4 ; GCN3-NEXT: .LBB17_2: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: .LBB17_3: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v3, s6 ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB17_4: ; %atomicrmw.end ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr double, ptr %out, i32 4 %result = atomicrmw xchg ptr %gep, double %in seq_cst ret double %result } define void @flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory(ptr %out, double %in) { ; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB18_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB18_4 ; GCN1-NEXT: .LBB18_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB18_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB18_2 ; GCN1-NEXT: .LBB18_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0 ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB18_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB18_4 ; GCN2-NEXT: .LBB18_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB18_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB18_2 ; GCN2-NEXT: .LBB18_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0 ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB18_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB18_4 ; GCN3-NEXT: .LBB18_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB18_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB18_2 ; GCN3-NEXT: .LBB18_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr double, ptr %out, i64 4 %tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define double @flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory(ptr %out, double %in) { ; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB19_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB19_4 ; GCN1-NEXT: .LBB19_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB19_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB19_2 ; GCN1-NEXT: .LBB19_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB19_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB19_4 ; GCN2-NEXT: .LBB19_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB19_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB19_2 ; GCN2-NEXT: .LBB19_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB19_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB19_4 ; GCN3-NEXT: .LBB19_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB19_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB19_2 ; GCN3-NEXT: .LBB19_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_nop 0 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr double, ptr %out, i64 4 %result = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0 ret double %result } ; --------------------------------------------------------------------- ; atomicrmw add ; --------------------------------------------------------------------- define void @flat_atomic_add_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_add_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB20_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB20_4 ; GCN1-NEXT: .LBB20_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB20_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB20_2 ; GCN1-NEXT: .LBB20_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_add_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB20_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB20_4 ; GCN2-NEXT: .LBB20_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB20_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB20_2 ; GCN2-NEXT: .LBB20_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_add_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB20_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB20_4 ; GCN3-NEXT: .LBB20_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB20_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB20_2 ; GCN3-NEXT: .LBB20_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_add_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_add_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB21_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB21_4 ; GCN1-NEXT: .LBB21_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB21_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB21_2 ; GCN1-NEXT: .LBB21_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_add_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB21_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB21_4 ; GCN2-NEXT: .LBB21_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB21_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB21_2 ; GCN2-NEXT: .LBB21_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_add_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB21_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB21_4 ; GCN3-NEXT: .LBB21_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB21_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB21_2 ; GCN3-NEXT: .LBB21_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_add_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_add_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_mov_b32_e32 v5, v1 ; GCN1-NEXT: v_mov_b32_e32 v4, v0 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB22_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB22_4 ; GCN1-NEXT: .LBB22_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB22_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB22_2 ; GCN1-NEXT: .LBB22_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_add_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_mov_b32_e32 v5, v1 ; GCN2-NEXT: v_mov_b32_e32 v4, v0 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB22_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB22_4 ; GCN2-NEXT: .LBB22_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB22_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB22_2 ; GCN2-NEXT: .LBB22_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_add_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB22_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB22_4 ; GCN3-NEXT: .LBB22_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB22_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB22_2 ; GCN3-NEXT: .LBB22_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw add ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_add_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_add_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB23_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB23_4 ; GCN1-NEXT: .LBB23_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB23_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB23_2 ; GCN1-NEXT: .LBB23_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_add_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB23_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB23_4 ; GCN2-NEXT: .LBB23_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB23_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB23_2 ; GCN2-NEXT: .LBB23_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_add_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB23_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB23_4 ; GCN3-NEXT: .LBB23_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB23_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB23_2 ; GCN3-NEXT: .LBB23_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw add ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_add_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_add_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB24_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB24_4 ; GCN1-NEXT: .LBB24_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB24_3: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v0, s4 ; GCN1-NEXT: v_mov_b32_e32 v2, s6 ; GCN1-NEXT: v_mov_b32_e32 v1, s5 ; GCN1-NEXT: v_mov_b32_e32 v3, s7 ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: s_cbranch_execnz .LBB24_2 ; GCN1-NEXT: .LBB24_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v1, vcc, s6, v1 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_add_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB24_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB24_4 ; GCN2-NEXT: .LBB24_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB24_3: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v0, s4 ; GCN2-NEXT: v_mov_b32_e32 v2, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s5 ; GCN2-NEXT: v_mov_b32_e32 v3, s7 ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: s_cbranch_execnz .LBB24_2 ; GCN2-NEXT: .LBB24_4: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v1, vcc, s6, v1 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_add_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB24_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB24_4 ; GCN3-NEXT: .LBB24_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB24_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v0, s4 ; GCN3-NEXT: v_mov_b32_e32 v2, s6 ; GCN3-NEXT: v_mov_b32_e32 v1, s5 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: s_cbranch_execnz .LBB24_2 ; GCN3-NEXT: .LBB24_4: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, s6, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v3, vcc ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_add_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_add_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB25_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB25_4 ; GCN1-NEXT: .LBB25_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB25_3: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v2, s6 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v3, s7 ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: s_cbranch_execnz .LBB25_2 ; GCN1-NEXT: .LBB25_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v1, vcc, s6, v1 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_add_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB25_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB25_4 ; GCN2-NEXT: .LBB25_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB25_3: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v2, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v3, s7 ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: s_cbranch_execnz .LBB25_2 ; GCN2-NEXT: .LBB25_4: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v1, vcc, s6, v1 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_add_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB25_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB25_4 ; GCN3-NEXT: .LBB25_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB25_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: v_mov_b32_e32 v2, s6 ; GCN3-NEXT: v_mov_b32_e32 v1, s35 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: s_cbranch_execnz .LBB25_2 ; GCN3-NEXT: .LBB25_4: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, s6, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v3, vcc ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_add_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_add_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB26_2 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v0, s4 ; GCN1-NEXT: v_mov_b32_e32 v2, s6 ; GCN1-NEXT: v_mov_b32_e32 v1, s5 ; GCN1-NEXT: v_mov_b32_e32 v3, s7 ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: s_cbranch_execz .LBB26_3 ; GCN1-NEXT: s_branch .LBB26_4 ; GCN1-NEXT: .LBB26_2: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: .LBB26_3: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v5, vcc, s6, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB26_4: ; %atomicrmw.end ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_add_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB26_2 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v0, s4 ; GCN2-NEXT: v_mov_b32_e32 v2, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s5 ; GCN2-NEXT: v_mov_b32_e32 v3, s7 ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: s_cbranch_execz .LBB26_3 ; GCN2-NEXT: s_branch .LBB26_4 ; GCN2-NEXT: .LBB26_2: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: .LBB26_3: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v5, vcc, s6, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB26_4: ; %atomicrmw.end ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_add_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB26_2 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v0, s4 ; GCN3-NEXT: v_mov_b32_e32 v2, s6 ; GCN3-NEXT: v_mov_b32_e32 v1, s5 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: s_cbranch_execz .LBB26_3 ; GCN3-NEXT: s_branch .LBB26_4 ; GCN3-NEXT: .LBB26_2: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: .LBB26_3: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, s6, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB26_4: ; %atomicrmw.end ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw add ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_add_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_add_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB27_2 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v2, s6 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v3, s7 ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: s_cbranch_execz .LBB27_3 ; GCN1-NEXT: s_branch .LBB27_4 ; GCN1-NEXT: .LBB27_2: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: .LBB27_3: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v5, vcc, s6, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB27_4: ; %atomicrmw.end ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_add_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB27_2 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v2, s6 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v3, s7 ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: s_cbranch_execz .LBB27_3 ; GCN2-NEXT: s_branch .LBB27_4 ; GCN2-NEXT: .LBB27_2: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: .LBB27_3: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v5, vcc, s6, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB27_4: ; %atomicrmw.end ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_add_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB27_2 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: v_mov_b32_e32 v2, s6 ; GCN3-NEXT: v_mov_b32_e32 v1, s35 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: s_cbranch_execz .LBB27_3 ; GCN3-NEXT: s_branch .LBB27_4 ; GCN3-NEXT: .LBB27_2: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: .LBB27_3: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, s6, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB27_4: ; %atomicrmw.end ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw add ptr %gep, i64 %in seq_cst ret i64 %result } define void @flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB28_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB28_4 ; GCN1-NEXT: .LBB28_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB28_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB28_2 ; GCN1-NEXT: .LBB28_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB28_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB28_4 ; GCN2-NEXT: .LBB28_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB28_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB28_2 ; GCN2-NEXT: .LBB28_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB28_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB28_4 ; GCN3-NEXT: .LBB28_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB28_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB28_2 ; GCN3-NEXT: .LBB28_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB29_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB29_4 ; GCN1-NEXT: .LBB29_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB29_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB29_2 ; GCN1-NEXT: .LBB29_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB29_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB29_4 ; GCN2-NEXT: .LBB29_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB29_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB29_2 ; GCN2-NEXT: .LBB29_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB29_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB29_4 ; GCN3-NEXT: .LBB29_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB29_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB29_2 ; GCN3-NEXT: .LBB29_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } ; --------------------------------------------------------------------- ; atomicrmw sub ; --------------------------------------------------------------------- define void @flat_atomic_sub_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_sub_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB30_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB30_6 ; GCN1-NEXT: .LBB30_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB30_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB30_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_sub_i32_e32 v4, vcc, v6, v2 ; GCN1-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB30_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB30_2 ; GCN1-NEXT: .LBB30_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_sub_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB30_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB30_6 ; GCN2-NEXT: .LBB30_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB30_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB30_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_sub_u32_e32 v4, vcc, v6, v2 ; GCN2-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB30_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB30_2 ; GCN2-NEXT: .LBB30_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_sub_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB30_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB30_6 ; GCN3-NEXT: .LBB30_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB30_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB30_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v2 ; GCN3-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB30_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB30_2 ; GCN3-NEXT: .LBB30_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_sub_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_sub_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB31_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB31_6 ; GCN1-NEXT: .LBB31_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB31_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB31_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_sub_i32_e32 v4, vcc, v6, v2 ; GCN1-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB31_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB31_2 ; GCN1-NEXT: .LBB31_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_sub_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB31_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB31_6 ; GCN2-NEXT: .LBB31_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB31_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB31_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_sub_u32_e32 v4, vcc, v6, v2 ; GCN2-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB31_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB31_2 ; GCN2-NEXT: .LBB31_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_sub_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB31_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB31_6 ; GCN3-NEXT: .LBB31_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB31_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB31_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v2 ; GCN3-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB31_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB31_2 ; GCN3-NEXT: .LBB31_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_sub_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_sub_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB32_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v5, v[4:5] ; GCN1-NEXT: flat_load_dword v4, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB32_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: v_sub_i32_e32 v4, vcc, v6, v2 ; GCN1-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB32_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: .LBB32_4: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB32_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v5, v3, vcc ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: .LBB32_6: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v0, v4 ; GCN1-NEXT: v_mov_b32_e32 v1, v5 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_sub_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB32_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v5, v[4:5] ; GCN2-NEXT: flat_load_dword v4, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB32_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: v_sub_u32_e32 v4, vcc, v6, v2 ; GCN2-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB32_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: .LBB32_4: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB32_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v5, v3, vcc ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: .LBB32_6: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v0, v4 ; GCN2-NEXT: v_mov_b32_e32 v1, v5 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_sub_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB32_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB32_6 ; GCN3-NEXT: .LBB32_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB32_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB32_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_sub_co_u32_e32 v6, vcc, v8, v2 ; GCN3-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v3, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB32_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB32_2 ; GCN3-NEXT: .LBB32_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw sub ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_sub_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_sub_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB33_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB33_6 ; GCN1-NEXT: .LBB33_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB33_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[4:5] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB33_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_sub_i32_e32 v6, vcc, v8, v2 ; GCN1-NEXT: v_subb_u32_e32 v7, vcc, v9, v3, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB33_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB33_2 ; GCN1-NEXT: .LBB33_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_sub_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB33_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB33_6 ; GCN2-NEXT: .LBB33_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB33_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[4:5] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB33_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_sub_u32_e32 v6, vcc, v8, v2 ; GCN2-NEXT: v_subb_u32_e32 v7, vcc, v9, v3, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB33_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB33_2 ; GCN2-NEXT: .LBB33_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_sub_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB33_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB33_6 ; GCN3-NEXT: .LBB33_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB33_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB33_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_sub_co_u32_e32 v6, vcc, v8, v2 ; GCN3-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v3, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB33_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB33_2 ; GCN3-NEXT: .LBB33_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw sub ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_sub_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB34_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccnz .LBB34_6 ; GCN1-NEXT: .LBB34_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB34_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v4, s4 ; GCN1-NEXT: v_mov_b32_e32 v5, s5 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s7 ; GCN1-NEXT: .LBB34_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, s6, v2 ; GCN1-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB34_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB34_2 ; GCN1-NEXT: .LBB34_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s6, v1 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_sub_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB34_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccnz .LBB34_6 ; GCN2-NEXT: .LBB34_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB34_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v4, s4 ; GCN2-NEXT: v_mov_b32_e32 v5, s5 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s7 ; GCN2-NEXT: .LBB34_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2 ; GCN2-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB34_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB34_2 ; GCN2-NEXT: .LBB34_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s6, v1 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_sub_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB34_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccnz .LBB34_6 ; GCN3-NEXT: .LBB34_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB34_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s4 ; GCN3-NEXT: v_mov_b32_e32 v5, s5 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s7 ; GCN3-NEXT: .LBB34_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2 ; GCN3-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v6, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB34_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB34_2 ; GCN3-NEXT: .LBB34_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_subrev_co_u32_e32 v1, vcc, s6, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_sub_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB35_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccnz .LBB35_6 ; GCN1-NEXT: .LBB35_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB35_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v4, s34 ; GCN1-NEXT: v_mov_b32_e32 v5, s35 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s7 ; GCN1-NEXT: .LBB35_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, s6, v2 ; GCN1-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB35_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB35_2 ; GCN1-NEXT: .LBB35_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s6, v1 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_sub_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB35_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccnz .LBB35_6 ; GCN2-NEXT: .LBB35_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB35_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v4, s34 ; GCN2-NEXT: v_mov_b32_e32 v5, s35 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s7 ; GCN2-NEXT: .LBB35_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2 ; GCN2-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB35_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB35_2 ; GCN2-NEXT: .LBB35_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s6, v1 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_sub_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB35_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccnz .LBB35_6 ; GCN3-NEXT: .LBB35_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB35_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s34 ; GCN3-NEXT: v_mov_b32_e32 v5, s35 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s7 ; GCN3-NEXT: .LBB35_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2 ; GCN3-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v6, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB35_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB35_2 ; GCN3-NEXT: .LBB35_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_subrev_co_u32_e32 v1, vcc, s6, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_sub_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB36_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v2, s4 ; GCN1-NEXT: v_mov_b32_e32 v3, s5 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: .LBB36_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v8, v1 ; GCN1-NEXT: v_mov_b32_e32 v7, v0 ; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v7 ; GCN1-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8] ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB36_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB36_6 ; GCN1-NEXT: .LBB36_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB36_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB36_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_sub_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB36_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v2, s4 ; GCN2-NEXT: v_mov_b32_e32 v3, s5 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: .LBB36_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v8, v1 ; GCN2-NEXT: v_mov_b32_e32 v7, v0 ; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v7 ; GCN2-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8] ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB36_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB36_6 ; GCN2-NEXT: .LBB36_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB36_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB36_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_sub_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB36_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s4 ; GCN3-NEXT: v_mov_b32_e32 v3, s5 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s7 ; GCN3-NEXT: .LBB36_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v8, v1 ; GCN3-NEXT: v_mov_b32_e32 v7, v0 ; GCN3-NEXT: v_subrev_co_u32_e32 v5, vcc, s6, v7 ; GCN3-NEXT: v_subb_co_u32_e32 v6, vcc, v8, v4, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8] ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB36_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB36_6 ; GCN3-NEXT: .LBB36_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB36_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB36_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw sub ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_sub_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB37_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: v_mov_b32_e32 v3, s35 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: .LBB37_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v8, v1 ; GCN1-NEXT: v_mov_b32_e32 v7, v0 ; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v7 ; GCN1-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8] ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB37_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB37_6 ; GCN1-NEXT: .LBB37_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB37_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB37_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_sub_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB37_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: v_mov_b32_e32 v3, s35 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: .LBB37_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v8, v1 ; GCN2-NEXT: v_mov_b32_e32 v7, v0 ; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v7 ; GCN2-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8] ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB37_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB37_6 ; GCN2-NEXT: .LBB37_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB37_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB37_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_sub_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB37_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: v_mov_b32_e32 v3, s35 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s7 ; GCN3-NEXT: .LBB37_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v8, v1 ; GCN3-NEXT: v_mov_b32_e32 v7, v0 ; GCN3-NEXT: v_subrev_co_u32_e32 v5, vcc, s6, v7 ; GCN3-NEXT: v_subb_co_u32_e32 v6, vcc, v8, v4, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8] ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB37_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB37_6 ; GCN3-NEXT: .LBB37_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB37_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB37_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw sub ptr %gep, i64 %in seq_cst ret i64 %result } define void @flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB38_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB38_4 ; GCN1-NEXT: .LBB38_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB38_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB38_2 ; GCN1-NEXT: .LBB38_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB38_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB38_4 ; GCN2-NEXT: .LBB38_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB38_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB38_2 ; GCN2-NEXT: .LBB38_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB38_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB38_4 ; GCN3-NEXT: .LBB38_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB38_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB38_2 ; GCN3-NEXT: .LBB38_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB39_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB39_4 ; GCN1-NEXT: .LBB39_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB39_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB39_2 ; GCN1-NEXT: .LBB39_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB39_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB39_4 ; GCN2-NEXT: .LBB39_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB39_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB39_2 ; GCN2-NEXT: .LBB39_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB39_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB39_4 ; GCN3-NEXT: .LBB39_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB39_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB39_2 ; GCN3-NEXT: .LBB39_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } ; --------------------------------------------------------------------- ; atomicrmw and ; --------------------------------------------------------------------- define void @flat_atomic_and_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_and_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB40_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB40_6 ; GCN1-NEXT: .LBB40_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB40_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB40_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v5, v7, v3 ; GCN1-NEXT: v_and_b32_e32 v4, v6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB40_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB40_2 ; GCN1-NEXT: .LBB40_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_and_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB40_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB40_6 ; GCN2-NEXT: .LBB40_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB40_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB40_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v5, v7, v3 ; GCN2-NEXT: v_and_b32_e32 v4, v6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB40_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB40_2 ; GCN2-NEXT: .LBB40_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_and_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB40_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB40_6 ; GCN3-NEXT: .LBB40_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB40_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB40_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v5, v7, v3 ; GCN3-NEXT: v_and_b32_e32 v4, v6, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB40_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB40_2 ; GCN3-NEXT: .LBB40_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v1, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_and_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_and_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB41_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB41_6 ; GCN1-NEXT: .LBB41_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB41_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB41_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v5, v7, v3 ; GCN1-NEXT: v_and_b32_e32 v4, v6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB41_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB41_2 ; GCN1-NEXT: .LBB41_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_and_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB41_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB41_6 ; GCN2-NEXT: .LBB41_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB41_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB41_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v5, v7, v3 ; GCN2-NEXT: v_and_b32_e32 v4, v6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB41_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB41_2 ; GCN2-NEXT: .LBB41_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_and_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB41_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB41_6 ; GCN3-NEXT: .LBB41_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB41_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB41_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v5, v7, v3 ; GCN3-NEXT: v_and_b32_e32 v4, v6, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB41_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB41_2 ; GCN3-NEXT: .LBB41_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v1, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_and_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_and_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB42_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v5, v[4:5] ; GCN1-NEXT: flat_load_dword v4, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB42_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: v_and_b32_e32 v5, v7, v3 ; GCN1-NEXT: v_and_b32_e32 v4, v6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB42_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: .LBB42_4: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB42_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: .LBB42_6: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v0, v4 ; GCN1-NEXT: v_mov_b32_e32 v1, v5 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_and_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB42_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v5, v[4:5] ; GCN2-NEXT: flat_load_dword v4, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB42_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: v_and_b32_e32 v5, v7, v3 ; GCN2-NEXT: v_and_b32_e32 v4, v6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB42_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: .LBB42_4: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB42_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: .LBB42_6: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v0, v4 ; GCN2-NEXT: v_mov_b32_e32 v1, v5 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_and_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB42_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB42_6 ; GCN3-NEXT: .LBB42_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB42_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB42_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_and_b32_e32 v7, v9, v3 ; GCN3-NEXT: v_and_b32_e32 v6, v8, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB42_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB42_2 ; GCN3-NEXT: .LBB42_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw and ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_and_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_and_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB43_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB43_6 ; GCN1-NEXT: .LBB43_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB43_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[4:5] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB43_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_and_b32_e32 v7, v9, v3 ; GCN1-NEXT: v_and_b32_e32 v6, v8, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB43_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB43_2 ; GCN1-NEXT: .LBB43_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_and_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB43_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB43_6 ; GCN2-NEXT: .LBB43_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB43_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[4:5] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB43_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_and_b32_e32 v7, v9, v3 ; GCN2-NEXT: v_and_b32_e32 v6, v8, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB43_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB43_2 ; GCN2-NEXT: .LBB43_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_and_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB43_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB43_6 ; GCN3-NEXT: .LBB43_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB43_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB43_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_and_b32_e32 v7, v9, v3 ; GCN3-NEXT: v_and_b32_e32 v6, v8, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB43_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB43_2 ; GCN3-NEXT: .LBB43_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw and ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_and_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB44_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccnz .LBB44_6 ; GCN1-NEXT: .LBB44_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB44_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v4, s4 ; GCN1-NEXT: v_mov_b32_e32 v5, s5 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: .LBB44_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v1, s7, v3 ; GCN1-NEXT: v_and_b32_e32 v0, s6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB44_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB44_2 ; GCN1-NEXT: .LBB44_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, s6, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, s7, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_and_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB44_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccnz .LBB44_6 ; GCN2-NEXT: .LBB44_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB44_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v4, s4 ; GCN2-NEXT: v_mov_b32_e32 v5, s5 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: .LBB44_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v1, s7, v3 ; GCN2-NEXT: v_and_b32_e32 v0, s6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB44_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB44_2 ; GCN2-NEXT: .LBB44_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, s6, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, s7, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_and_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB44_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccnz .LBB44_6 ; GCN3-NEXT: .LBB44_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB44_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s4 ; GCN3-NEXT: v_mov_b32_e32 v5, s5 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: .LBB44_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v1, s7, v3 ; GCN3-NEXT: v_and_b32_e32 v0, s6, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB44_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB44_2 ; GCN3-NEXT: .LBB44_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v1, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, s6, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_and_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB45_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccnz .LBB45_6 ; GCN1-NEXT: .LBB45_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB45_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v4, s34 ; GCN1-NEXT: v_mov_b32_e32 v5, s35 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: .LBB45_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v1, s7, v3 ; GCN1-NEXT: v_and_b32_e32 v0, s6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB45_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB45_2 ; GCN1-NEXT: .LBB45_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, s6, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, s7, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_and_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB45_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccnz .LBB45_6 ; GCN2-NEXT: .LBB45_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB45_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v4, s34 ; GCN2-NEXT: v_mov_b32_e32 v5, s35 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: .LBB45_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v1, s7, v3 ; GCN2-NEXT: v_and_b32_e32 v0, s6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB45_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB45_2 ; GCN2-NEXT: .LBB45_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, s6, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, s7, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_and_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB45_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccnz .LBB45_6 ; GCN3-NEXT: .LBB45_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB45_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s34 ; GCN3-NEXT: v_mov_b32_e32 v5, s35 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: .LBB45_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v1, s7, v3 ; GCN3-NEXT: v_and_b32_e32 v0, s6, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB45_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB45_2 ; GCN3-NEXT: .LBB45_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v1, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, s6, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_and_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB46_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v2, s4 ; GCN1-NEXT: v_mov_b32_e32 v3, s5 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: .LBB46_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v1 ; GCN1-NEXT: v_mov_b32_e32 v6, v0 ; GCN1-NEXT: v_and_b32_e32 v5, s7, v7 ; GCN1-NEXT: v_and_b32_e32 v4, s6, v6 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB46_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB46_6 ; GCN1-NEXT: .LBB46_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB46_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v4, s6, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v5, s7, v1 ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB46_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_and_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB46_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v2, s4 ; GCN2-NEXT: v_mov_b32_e32 v3, s5 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: .LBB46_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v1 ; GCN2-NEXT: v_mov_b32_e32 v6, v0 ; GCN2-NEXT: v_and_b32_e32 v5, s7, v7 ; GCN2-NEXT: v_and_b32_e32 v4, s6, v6 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB46_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB46_6 ; GCN2-NEXT: .LBB46_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB46_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v4, s6, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v5, s7, v1 ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB46_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_and_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB46_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s4 ; GCN3-NEXT: v_mov_b32_e32 v3, s5 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: .LBB46_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v7, v1 ; GCN3-NEXT: v_mov_b32_e32 v6, v0 ; GCN3-NEXT: v_and_b32_e32 v5, s7, v7 ; GCN3-NEXT: v_and_b32_e32 v4, s6, v6 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB46_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB46_6 ; GCN3-NEXT: .LBB46_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB46_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v3, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v4, s6, v0 ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB46_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw and ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_and_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB47_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: v_mov_b32_e32 v3, s35 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: .LBB47_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v1 ; GCN1-NEXT: v_mov_b32_e32 v6, v0 ; GCN1-NEXT: v_and_b32_e32 v5, s7, v7 ; GCN1-NEXT: v_and_b32_e32 v4, s6, v6 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB47_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB47_6 ; GCN1-NEXT: .LBB47_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB47_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v4, s6, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v5, s7, v1 ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB47_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_and_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB47_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: v_mov_b32_e32 v3, s35 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: .LBB47_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v1 ; GCN2-NEXT: v_mov_b32_e32 v6, v0 ; GCN2-NEXT: v_and_b32_e32 v5, s7, v7 ; GCN2-NEXT: v_and_b32_e32 v4, s6, v6 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB47_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB47_6 ; GCN2-NEXT: .LBB47_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB47_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v4, s6, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v5, s7, v1 ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB47_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_and_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB47_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: v_mov_b32_e32 v3, s35 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: .LBB47_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v7, v1 ; GCN3-NEXT: v_mov_b32_e32 v6, v0 ; GCN3-NEXT: v_and_b32_e32 v5, s7, v7 ; GCN3-NEXT: v_and_b32_e32 v4, s6, v6 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB47_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB47_6 ; GCN3-NEXT: .LBB47_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB47_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v3, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v4, s6, v0 ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB47_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw and ptr %gep, i64 %in seq_cst ret i64 %result } define void @flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB48_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB48_4 ; GCN1-NEXT: .LBB48_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB48_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB48_2 ; GCN1-NEXT: .LBB48_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB48_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB48_4 ; GCN2-NEXT: .LBB48_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB48_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB48_2 ; GCN2-NEXT: .LBB48_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB48_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB48_4 ; GCN3-NEXT: .LBB48_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB48_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB48_2 ; GCN3-NEXT: .LBB48_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v1, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB49_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB49_4 ; GCN1-NEXT: .LBB49_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB49_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB49_2 ; GCN1-NEXT: .LBB49_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB49_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB49_4 ; GCN2-NEXT: .LBB49_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB49_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB49_2 ; GCN2-NEXT: .LBB49_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB49_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB49_4 ; GCN3-NEXT: .LBB49_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB49_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB49_2 ; GCN3-NEXT: .LBB49_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } ; --------------------------------------------------------------------- ; atomicrmw nand ; --------------------------------------------------------------------- define void @flat_atomic_nand_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_nand_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB50_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB50_6 ; GCN1-NEXT: .LBB50_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB50_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB50_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v4, v7, v3 ; GCN1-NEXT: v_and_b32_e32 v8, v6, v2 ; GCN1-NEXT: v_not_b32_e32 v5, v4 ; GCN1-NEXT: v_not_b32_e32 v4, v8 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB50_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB50_2 ; GCN1-NEXT: .LBB50_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN1-NEXT: v_not_b32_e32 v2, v2 ; GCN1-NEXT: v_not_b32_e32 v3, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_nand_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB50_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB50_6 ; GCN2-NEXT: .LBB50_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB50_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB50_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v4, v7, v3 ; GCN2-NEXT: v_and_b32_e32 v8, v6, v2 ; GCN2-NEXT: v_not_b32_e32 v5, v4 ; GCN2-NEXT: v_not_b32_e32 v4, v8 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB50_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB50_2 ; GCN2-NEXT: .LBB50_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN2-NEXT: v_not_b32_e32 v2, v2 ; GCN2-NEXT: v_not_b32_e32 v3, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_nand_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB50_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB50_6 ; GCN3-NEXT: .LBB50_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB50_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB50_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v4, v7, v3 ; GCN3-NEXT: v_and_b32_e32 v8, v6, v2 ; GCN3-NEXT: v_not_b32_e32 v5, v4 ; GCN3-NEXT: v_not_b32_e32 v4, v8 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB50_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB50_2 ; GCN3-NEXT: .LBB50_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v1, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN3-NEXT: v_not_b32_e32 v2, v2 ; GCN3-NEXT: v_not_b32_e32 v1, v1 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_nand_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_nand_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB51_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB51_6 ; GCN1-NEXT: .LBB51_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB51_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB51_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v4, v7, v3 ; GCN1-NEXT: v_and_b32_e32 v8, v6, v2 ; GCN1-NEXT: v_not_b32_e32 v5, v4 ; GCN1-NEXT: v_not_b32_e32 v4, v8 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB51_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB51_2 ; GCN1-NEXT: .LBB51_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN1-NEXT: v_not_b32_e32 v2, v2 ; GCN1-NEXT: v_not_b32_e32 v3, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_nand_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB51_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB51_6 ; GCN2-NEXT: .LBB51_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB51_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB51_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v4, v7, v3 ; GCN2-NEXT: v_and_b32_e32 v8, v6, v2 ; GCN2-NEXT: v_not_b32_e32 v5, v4 ; GCN2-NEXT: v_not_b32_e32 v4, v8 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB51_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB51_2 ; GCN2-NEXT: .LBB51_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN2-NEXT: v_not_b32_e32 v2, v2 ; GCN2-NEXT: v_not_b32_e32 v3, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_nand_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB51_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB51_6 ; GCN3-NEXT: .LBB51_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB51_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB51_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v4, v7, v3 ; GCN3-NEXT: v_and_b32_e32 v8, v6, v2 ; GCN3-NEXT: v_not_b32_e32 v5, v4 ; GCN3-NEXT: v_not_b32_e32 v4, v8 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB51_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB51_2 ; GCN3-NEXT: .LBB51_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v1, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN3-NEXT: v_not_b32_e32 v2, v2 ; GCN3-NEXT: v_not_b32_e32 v1, v1 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_nand_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_nand_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB52_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v5, v[4:5] ; GCN1-NEXT: flat_load_dword v4, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB52_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: v_and_b32_e32 v4, v7, v3 ; GCN1-NEXT: v_and_b32_e32 v8, v6, v2 ; GCN1-NEXT: v_not_b32_e32 v5, v4 ; GCN1-NEXT: v_not_b32_e32 v4, v8 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB52_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: .LBB52_4: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB52_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN1-NEXT: v_not_b32_e32 v2, v2 ; GCN1-NEXT: v_not_b32_e32 v3, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: .LBB52_6: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v0, v4 ; GCN1-NEXT: v_mov_b32_e32 v1, v5 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_nand_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB52_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v5, v[4:5] ; GCN2-NEXT: flat_load_dword v4, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB52_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: v_and_b32_e32 v4, v7, v3 ; GCN2-NEXT: v_and_b32_e32 v8, v6, v2 ; GCN2-NEXT: v_not_b32_e32 v5, v4 ; GCN2-NEXT: v_not_b32_e32 v4, v8 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB52_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: .LBB52_4: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB52_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN2-NEXT: v_not_b32_e32 v2, v2 ; GCN2-NEXT: v_not_b32_e32 v3, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: .LBB52_6: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v0, v4 ; GCN2-NEXT: v_mov_b32_e32 v1, v5 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_nand_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB52_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB52_6 ; GCN3-NEXT: .LBB52_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB52_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB52_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_and_b32_e32 v0, v9, v3 ; GCN3-NEXT: v_and_b32_e32 v1, v8, v2 ; GCN3-NEXT: v_not_b32_e32 v7, v0 ; GCN3-NEXT: v_not_b32_e32 v6, v1 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB52_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB52_2 ; GCN3-NEXT: .LBB52_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN3-NEXT: v_not_b32_e32 v2, v2 ; GCN3-NEXT: v_not_b32_e32 v3, v3 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw nand ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_nand_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_nand_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB53_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB53_6 ; GCN1-NEXT: .LBB53_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB53_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[4:5] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB53_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_and_b32_e32 v0, v9, v3 ; GCN1-NEXT: v_and_b32_e32 v1, v8, v2 ; GCN1-NEXT: v_not_b32_e32 v7, v0 ; GCN1-NEXT: v_not_b32_e32 v6, v1 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB53_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB53_2 ; GCN1-NEXT: .LBB53_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN1-NEXT: v_not_b32_e32 v2, v2 ; GCN1-NEXT: v_not_b32_e32 v3, v3 ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_nand_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB53_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB53_6 ; GCN2-NEXT: .LBB53_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB53_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[4:5] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB53_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_and_b32_e32 v0, v9, v3 ; GCN2-NEXT: v_and_b32_e32 v1, v8, v2 ; GCN2-NEXT: v_not_b32_e32 v7, v0 ; GCN2-NEXT: v_not_b32_e32 v6, v1 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB53_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB53_2 ; GCN2-NEXT: .LBB53_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN2-NEXT: v_not_b32_e32 v2, v2 ; GCN2-NEXT: v_not_b32_e32 v3, v3 ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_nand_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB53_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB53_6 ; GCN3-NEXT: .LBB53_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB53_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB53_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_and_b32_e32 v0, v9, v3 ; GCN3-NEXT: v_and_b32_e32 v1, v8, v2 ; GCN3-NEXT: v_not_b32_e32 v7, v0 ; GCN3-NEXT: v_not_b32_e32 v6, v1 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB53_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB53_2 ; GCN3-NEXT: .LBB53_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN3-NEXT: v_not_b32_e32 v2, v2 ; GCN3-NEXT: v_not_b32_e32 v3, v3 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw nand ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_nand_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB54_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccnz .LBB54_6 ; GCN1-NEXT: .LBB54_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB54_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v4, s4 ; GCN1-NEXT: v_mov_b32_e32 v5, s5 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: .LBB54_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v0, s7, v3 ; GCN1-NEXT: v_and_b32_e32 v6, s6, v2 ; GCN1-NEXT: v_not_b32_e32 v1, v0 ; GCN1-NEXT: v_not_b32_e32 v0, v6 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB54_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB54_2 ; GCN1-NEXT: .LBB54_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, s6, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, s7, v3 ; GCN1-NEXT: v_not_b32_e32 v2, v2 ; GCN1-NEXT: v_not_b32_e32 v3, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_nand_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB54_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccnz .LBB54_6 ; GCN2-NEXT: .LBB54_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB54_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v4, s4 ; GCN2-NEXT: v_mov_b32_e32 v5, s5 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: .LBB54_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v0, s7, v3 ; GCN2-NEXT: v_and_b32_e32 v6, s6, v2 ; GCN2-NEXT: v_not_b32_e32 v1, v0 ; GCN2-NEXT: v_not_b32_e32 v0, v6 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB54_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB54_2 ; GCN2-NEXT: .LBB54_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, s6, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, s7, v3 ; GCN2-NEXT: v_not_b32_e32 v2, v2 ; GCN2-NEXT: v_not_b32_e32 v3, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_nand_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB54_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccnz .LBB54_6 ; GCN3-NEXT: .LBB54_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB54_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s4 ; GCN3-NEXT: v_mov_b32_e32 v5, s5 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: .LBB54_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v0, s7, v3 ; GCN3-NEXT: v_and_b32_e32 v6, s6, v2 ; GCN3-NEXT: v_not_b32_e32 v1, v0 ; GCN3-NEXT: v_not_b32_e32 v0, v6 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB54_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB54_2 ; GCN3-NEXT: .LBB54_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v1, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, s6, v2 ; GCN3-NEXT: v_not_b32_e32 v2, v2 ; GCN3-NEXT: v_not_b32_e32 v1, v1 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_nand_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB55_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccnz .LBB55_6 ; GCN1-NEXT: .LBB55_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB55_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v4, s34 ; GCN1-NEXT: v_mov_b32_e32 v5, s35 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: .LBB55_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v0, s7, v3 ; GCN1-NEXT: v_and_b32_e32 v6, s6, v2 ; GCN1-NEXT: v_not_b32_e32 v1, v0 ; GCN1-NEXT: v_not_b32_e32 v0, v6 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB55_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB55_2 ; GCN1-NEXT: .LBB55_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, s6, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, s7, v3 ; GCN1-NEXT: v_not_b32_e32 v2, v2 ; GCN1-NEXT: v_not_b32_e32 v3, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_nand_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB55_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccnz .LBB55_6 ; GCN2-NEXT: .LBB55_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB55_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v4, s34 ; GCN2-NEXT: v_mov_b32_e32 v5, s35 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: .LBB55_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v0, s7, v3 ; GCN2-NEXT: v_and_b32_e32 v6, s6, v2 ; GCN2-NEXT: v_not_b32_e32 v1, v0 ; GCN2-NEXT: v_not_b32_e32 v0, v6 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB55_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB55_2 ; GCN2-NEXT: .LBB55_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, s6, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, s7, v3 ; GCN2-NEXT: v_not_b32_e32 v2, v2 ; GCN2-NEXT: v_not_b32_e32 v3, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_nand_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB55_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccnz .LBB55_6 ; GCN3-NEXT: .LBB55_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB55_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s34 ; GCN3-NEXT: v_mov_b32_e32 v5, s35 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: .LBB55_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v0, s7, v3 ; GCN3-NEXT: v_and_b32_e32 v6, s6, v2 ; GCN3-NEXT: v_not_b32_e32 v1, v0 ; GCN3-NEXT: v_not_b32_e32 v0, v6 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB55_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB55_2 ; GCN3-NEXT: .LBB55_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v1, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, s6, v2 ; GCN3-NEXT: v_not_b32_e32 v2, v2 ; GCN3-NEXT: v_not_b32_e32 v1, v1 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_nand_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB56_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v2, s4 ; GCN1-NEXT: v_mov_b32_e32 v3, s5 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: .LBB56_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v1 ; GCN1-NEXT: v_mov_b32_e32 v6, v0 ; GCN1-NEXT: v_and_b32_e32 v0, s7, v7 ; GCN1-NEXT: v_and_b32_e32 v1, s6, v6 ; GCN1-NEXT: v_not_b32_e32 v5, v0 ; GCN1-NEXT: v_not_b32_e32 v4, v1 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB56_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB56_6 ; GCN1-NEXT: .LBB56_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB56_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v4, s6, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v5, s7, v1 ; GCN1-NEXT: v_not_b32_e32 v4, v4 ; GCN1-NEXT: v_not_b32_e32 v5, v5 ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB56_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_nand_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB56_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v2, s4 ; GCN2-NEXT: v_mov_b32_e32 v3, s5 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: .LBB56_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v1 ; GCN2-NEXT: v_mov_b32_e32 v6, v0 ; GCN2-NEXT: v_and_b32_e32 v0, s7, v7 ; GCN2-NEXT: v_and_b32_e32 v1, s6, v6 ; GCN2-NEXT: v_not_b32_e32 v5, v0 ; GCN2-NEXT: v_not_b32_e32 v4, v1 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB56_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB56_6 ; GCN2-NEXT: .LBB56_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB56_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v4, s6, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v5, s7, v1 ; GCN2-NEXT: v_not_b32_e32 v4, v4 ; GCN2-NEXT: v_not_b32_e32 v5, v5 ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB56_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_nand_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB56_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s4 ; GCN3-NEXT: v_mov_b32_e32 v3, s5 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: .LBB56_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v7, v1 ; GCN3-NEXT: v_mov_b32_e32 v6, v0 ; GCN3-NEXT: v_and_b32_e32 v0, s7, v7 ; GCN3-NEXT: v_and_b32_e32 v1, s6, v6 ; GCN3-NEXT: v_not_b32_e32 v5, v0 ; GCN3-NEXT: v_not_b32_e32 v4, v1 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB56_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB56_6 ; GCN3-NEXT: .LBB56_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB56_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v3, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v4, s6, v0 ; GCN3-NEXT: v_not_b32_e32 v4, v4 ; GCN3-NEXT: v_not_b32_e32 v3, v3 ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB56_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw nand ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_nand_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB57_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: v_mov_b32_e32 v3, s35 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: .LBB57_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v1 ; GCN1-NEXT: v_mov_b32_e32 v6, v0 ; GCN1-NEXT: v_and_b32_e32 v0, s7, v7 ; GCN1-NEXT: v_and_b32_e32 v1, s6, v6 ; GCN1-NEXT: v_not_b32_e32 v5, v0 ; GCN1-NEXT: v_not_b32_e32 v4, v1 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB57_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB57_6 ; GCN1-NEXT: .LBB57_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB57_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v4, s6, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v5, s7, v1 ; GCN1-NEXT: v_not_b32_e32 v4, v4 ; GCN1-NEXT: v_not_b32_e32 v5, v5 ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB57_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_nand_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB57_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: v_mov_b32_e32 v3, s35 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: .LBB57_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v1 ; GCN2-NEXT: v_mov_b32_e32 v6, v0 ; GCN2-NEXT: v_and_b32_e32 v0, s7, v7 ; GCN2-NEXT: v_and_b32_e32 v1, s6, v6 ; GCN2-NEXT: v_not_b32_e32 v5, v0 ; GCN2-NEXT: v_not_b32_e32 v4, v1 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB57_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB57_6 ; GCN2-NEXT: .LBB57_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB57_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v4, s6, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v5, s7, v1 ; GCN2-NEXT: v_not_b32_e32 v4, v4 ; GCN2-NEXT: v_not_b32_e32 v5, v5 ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB57_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_nand_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB57_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: v_mov_b32_e32 v3, s35 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: .LBB57_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v7, v1 ; GCN3-NEXT: v_mov_b32_e32 v6, v0 ; GCN3-NEXT: v_and_b32_e32 v0, s7, v7 ; GCN3-NEXT: v_and_b32_e32 v1, s6, v6 ; GCN3-NEXT: v_not_b32_e32 v5, v0 ; GCN3-NEXT: v_not_b32_e32 v4, v1 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB57_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB57_6 ; GCN3-NEXT: .LBB57_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB57_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v3, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v4, s6, v0 ; GCN3-NEXT: v_not_b32_e32 v4, v4 ; GCN3-NEXT: v_not_b32_e32 v3, v3 ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB57_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw nand ptr %gep, i64 %in seq_cst ret i64 %result } define void @flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB58_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB58_6 ; GCN1-NEXT: .LBB58_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB58_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB58_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v4, v7, v3 ; GCN1-NEXT: v_and_b32_e32 v8, v6, v2 ; GCN1-NEXT: v_not_b32_e32 v5, v4 ; GCN1-NEXT: v_not_b32_e32 v4, v8 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB58_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB58_2 ; GCN1-NEXT: .LBB58_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN1-NEXT: v_not_b32_e32 v2, v2 ; GCN1-NEXT: v_not_b32_e32 v3, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB58_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB58_6 ; GCN2-NEXT: .LBB58_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB58_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB58_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v4, v7, v3 ; GCN2-NEXT: v_and_b32_e32 v8, v6, v2 ; GCN2-NEXT: v_not_b32_e32 v5, v4 ; GCN2-NEXT: v_not_b32_e32 v4, v8 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB58_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB58_2 ; GCN2-NEXT: .LBB58_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 ; GCN2-NEXT: v_not_b32_e32 v2, v2 ; GCN2-NEXT: v_not_b32_e32 v3, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB58_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB58_6 ; GCN3-NEXT: .LBB58_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB58_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB58_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v4, v7, v3 ; GCN3-NEXT: v_and_b32_e32 v8, v6, v2 ; GCN3-NEXT: v_not_b32_e32 v5, v4 ; GCN3-NEXT: v_not_b32_e32 v4, v8 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB58_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB58_2 ; GCN3-NEXT: .LBB58_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v1, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, v4, v2 ; GCN3-NEXT: v_not_b32_e32 v2, v2 ; GCN3-NEXT: v_not_b32_e32 v1, v1 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB59_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB59_6 ; GCN1-NEXT: .LBB59_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB59_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[4:5] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB59_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_and_b32_e32 v0, v9, v3 ; GCN1-NEXT: v_and_b32_e32 v1, v8, v2 ; GCN1-NEXT: v_not_b32_e32 v7, v0 ; GCN1-NEXT: v_not_b32_e32 v6, v1 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB59_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB59_2 ; GCN1-NEXT: .LBB59_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN1-NEXT: v_not_b32_e32 v2, v2 ; GCN1-NEXT: v_not_b32_e32 v3, v3 ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB59_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB59_6 ; GCN2-NEXT: .LBB59_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB59_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[4:5] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB59_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_and_b32_e32 v0, v9, v3 ; GCN2-NEXT: v_and_b32_e32 v1, v8, v2 ; GCN2-NEXT: v_not_b32_e32 v7, v0 ; GCN2-NEXT: v_not_b32_e32 v6, v1 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB59_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB59_2 ; GCN2-NEXT: .LBB59_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN2-NEXT: v_not_b32_e32 v2, v2 ; GCN2-NEXT: v_not_b32_e32 v3, v3 ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB59_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB59_6 ; GCN3-NEXT: .LBB59_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB59_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB59_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_and_b32_e32 v0, v9, v3 ; GCN3-NEXT: v_and_b32_e32 v1, v8, v2 ; GCN3-NEXT: v_not_b32_e32 v7, v0 ; GCN3-NEXT: v_not_b32_e32 v6, v1 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB59_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB59_2 ; GCN3-NEXT: .LBB59_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_and_b32_e32 v3, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_and_b32_e32 v2, v0, v2 ; GCN3-NEXT: v_not_b32_e32 v2, v2 ; GCN3-NEXT: v_not_b32_e32 v3, v3 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } ; --------------------------------------------------------------------- ; atomicrmw or ; --------------------------------------------------------------------- define void @flat_atomic_or_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_or_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB60_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB60_6 ; GCN1-NEXT: .LBB60_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB60_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB60_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v5, v7, v3 ; GCN1-NEXT: v_or_b32_e32 v4, v6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB60_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB60_2 ; GCN1-NEXT: .LBB60_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_or_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v3, v5, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_or_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB60_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB60_6 ; GCN2-NEXT: .LBB60_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB60_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB60_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v5, v7, v3 ; GCN2-NEXT: v_or_b32_e32 v4, v6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB60_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB60_2 ; GCN2-NEXT: .LBB60_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_or_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v3, v5, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_or_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB60_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB60_6 ; GCN3-NEXT: .LBB60_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB60_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB60_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v5, v7, v3 ; GCN3-NEXT: v_or_b32_e32 v4, v6, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB60_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB60_2 ; GCN3-NEXT: .LBB60_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_or_b32_e32 v1, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v2, v4, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_or_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_or_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB61_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB61_6 ; GCN1-NEXT: .LBB61_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB61_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB61_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v5, v7, v3 ; GCN1-NEXT: v_or_b32_e32 v4, v6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB61_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB61_2 ; GCN1-NEXT: .LBB61_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_or_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v3, v5, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_or_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB61_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB61_6 ; GCN2-NEXT: .LBB61_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB61_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB61_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v5, v7, v3 ; GCN2-NEXT: v_or_b32_e32 v4, v6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB61_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB61_2 ; GCN2-NEXT: .LBB61_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_or_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v3, v5, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_or_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB61_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB61_6 ; GCN3-NEXT: .LBB61_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB61_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB61_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v5, v7, v3 ; GCN3-NEXT: v_or_b32_e32 v4, v6, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB61_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB61_2 ; GCN3-NEXT: .LBB61_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_or_b32_e32 v1, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v2, v4, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_or_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_or_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB62_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v5, v[4:5] ; GCN1-NEXT: flat_load_dword v4, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB62_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: v_or_b32_e32 v5, v7, v3 ; GCN1-NEXT: v_or_b32_e32 v4, v6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB62_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: .LBB62_4: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB62_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_or_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v3, v5, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: .LBB62_6: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v0, v4 ; GCN1-NEXT: v_mov_b32_e32 v1, v5 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_or_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB62_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v5, v[4:5] ; GCN2-NEXT: flat_load_dword v4, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB62_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: v_or_b32_e32 v5, v7, v3 ; GCN2-NEXT: v_or_b32_e32 v4, v6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB62_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: .LBB62_4: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB62_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_or_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v3, v5, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: .LBB62_6: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v0, v4 ; GCN2-NEXT: v_mov_b32_e32 v1, v5 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_or_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB62_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB62_6 ; GCN3-NEXT: .LBB62_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB62_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB62_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_or_b32_e32 v7, v9, v3 ; GCN3-NEXT: v_or_b32_e32 v6, v8, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB62_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB62_2 ; GCN3-NEXT: .LBB62_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_or_b32_e32 v3, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v2, v0, v2 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw or ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_or_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_or_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB63_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB63_6 ; GCN1-NEXT: .LBB63_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB63_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[4:5] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB63_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_or_b32_e32 v7, v9, v3 ; GCN1-NEXT: v_or_b32_e32 v6, v8, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB63_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB63_2 ; GCN1-NEXT: .LBB63_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_or_b32_e32 v2, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v3, v1, v3 ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_or_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB63_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB63_6 ; GCN2-NEXT: .LBB63_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB63_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[4:5] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB63_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_or_b32_e32 v7, v9, v3 ; GCN2-NEXT: v_or_b32_e32 v6, v8, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB63_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB63_2 ; GCN2-NEXT: .LBB63_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_or_b32_e32 v2, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v3, v1, v3 ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_or_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB63_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB63_6 ; GCN3-NEXT: .LBB63_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB63_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB63_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_or_b32_e32 v7, v9, v3 ; GCN3-NEXT: v_or_b32_e32 v6, v8, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB63_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB63_2 ; GCN3-NEXT: .LBB63_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_or_b32_e32 v3, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v2, v0, v2 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw or ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_or_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB64_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccnz .LBB64_6 ; GCN1-NEXT: .LBB64_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB64_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v4, s4 ; GCN1-NEXT: v_mov_b32_e32 v5, s5 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: .LBB64_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v1, s7, v3 ; GCN1-NEXT: v_or_b32_e32 v0, s6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB64_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB64_2 ; GCN1-NEXT: .LBB64_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_or_b32_e32 v2, s6, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v3, s7, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_or_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB64_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccnz .LBB64_6 ; GCN2-NEXT: .LBB64_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB64_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v4, s4 ; GCN2-NEXT: v_mov_b32_e32 v5, s5 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: .LBB64_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v1, s7, v3 ; GCN2-NEXT: v_or_b32_e32 v0, s6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB64_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB64_2 ; GCN2-NEXT: .LBB64_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_or_b32_e32 v2, s6, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v3, s7, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_or_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB64_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccnz .LBB64_6 ; GCN3-NEXT: .LBB64_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB64_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s4 ; GCN3-NEXT: v_mov_b32_e32 v5, s5 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: .LBB64_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v1, s7, v3 ; GCN3-NEXT: v_or_b32_e32 v0, s6, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB64_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB64_2 ; GCN3-NEXT: .LBB64_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_or_b32_e32 v1, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v2, s6, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_or_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB65_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccnz .LBB65_6 ; GCN1-NEXT: .LBB65_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB65_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v4, s34 ; GCN1-NEXT: v_mov_b32_e32 v5, s35 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: .LBB65_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v1, s7, v3 ; GCN1-NEXT: v_or_b32_e32 v0, s6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB65_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB65_2 ; GCN1-NEXT: .LBB65_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_or_b32_e32 v2, s6, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v3, s7, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_or_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB65_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccnz .LBB65_6 ; GCN2-NEXT: .LBB65_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB65_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v4, s34 ; GCN2-NEXT: v_mov_b32_e32 v5, s35 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: .LBB65_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v1, s7, v3 ; GCN2-NEXT: v_or_b32_e32 v0, s6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB65_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB65_2 ; GCN2-NEXT: .LBB65_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_or_b32_e32 v2, s6, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v3, s7, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_or_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB65_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccnz .LBB65_6 ; GCN3-NEXT: .LBB65_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB65_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s34 ; GCN3-NEXT: v_mov_b32_e32 v5, s35 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: .LBB65_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v1, s7, v3 ; GCN3-NEXT: v_or_b32_e32 v0, s6, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB65_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB65_2 ; GCN3-NEXT: .LBB65_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_or_b32_e32 v1, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v2, s6, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_or_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB66_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v2, s4 ; GCN1-NEXT: v_mov_b32_e32 v3, s5 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: .LBB66_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v1 ; GCN1-NEXT: v_mov_b32_e32 v6, v0 ; GCN1-NEXT: v_or_b32_e32 v5, s7, v7 ; GCN1-NEXT: v_or_b32_e32 v4, s6, v6 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB66_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB66_6 ; GCN1-NEXT: .LBB66_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB66_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_or_b32_e32 v4, s6, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v5, s7, v1 ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB66_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_or_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB66_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v2, s4 ; GCN2-NEXT: v_mov_b32_e32 v3, s5 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: .LBB66_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v1 ; GCN2-NEXT: v_mov_b32_e32 v6, v0 ; GCN2-NEXT: v_or_b32_e32 v5, s7, v7 ; GCN2-NEXT: v_or_b32_e32 v4, s6, v6 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB66_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB66_6 ; GCN2-NEXT: .LBB66_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB66_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_or_b32_e32 v4, s6, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v5, s7, v1 ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB66_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_or_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB66_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s4 ; GCN3-NEXT: v_mov_b32_e32 v3, s5 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: .LBB66_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v7, v1 ; GCN3-NEXT: v_mov_b32_e32 v6, v0 ; GCN3-NEXT: v_or_b32_e32 v5, s7, v7 ; GCN3-NEXT: v_or_b32_e32 v4, s6, v6 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB66_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB66_6 ; GCN3-NEXT: .LBB66_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB66_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_or_b32_e32 v3, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v4, s6, v0 ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB66_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw or ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_or_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB67_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: v_mov_b32_e32 v3, s35 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: .LBB67_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v1 ; GCN1-NEXT: v_mov_b32_e32 v6, v0 ; GCN1-NEXT: v_or_b32_e32 v5, s7, v7 ; GCN1-NEXT: v_or_b32_e32 v4, s6, v6 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB67_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB67_6 ; GCN1-NEXT: .LBB67_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB67_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_or_b32_e32 v4, s6, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v5, s7, v1 ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB67_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_or_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB67_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: v_mov_b32_e32 v3, s35 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: .LBB67_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v1 ; GCN2-NEXT: v_mov_b32_e32 v6, v0 ; GCN2-NEXT: v_or_b32_e32 v5, s7, v7 ; GCN2-NEXT: v_or_b32_e32 v4, s6, v6 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB67_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB67_6 ; GCN2-NEXT: .LBB67_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB67_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_or_b32_e32 v4, s6, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v5, s7, v1 ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB67_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_or_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB67_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: v_mov_b32_e32 v3, s35 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: .LBB67_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v7, v1 ; GCN3-NEXT: v_mov_b32_e32 v6, v0 ; GCN3-NEXT: v_or_b32_e32 v5, s7, v7 ; GCN3-NEXT: v_or_b32_e32 v4, s6, v6 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB67_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB67_6 ; GCN3-NEXT: .LBB67_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB67_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_or_b32_e32 v3, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v4, s6, v0 ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB67_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw or ptr %gep, i64 %in seq_cst ret i64 %result } define void @flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB68_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB68_4 ; GCN1-NEXT: .LBB68_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB68_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB68_2 ; GCN1-NEXT: .LBB68_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_or_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v3, v5, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB68_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB68_4 ; GCN2-NEXT: .LBB68_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB68_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB68_2 ; GCN2-NEXT: .LBB68_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_or_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v3, v5, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB68_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB68_4 ; GCN3-NEXT: .LBB68_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB68_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB68_2 ; GCN3-NEXT: .LBB68_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_or_b32_e32 v1, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v2, v4, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB69_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB69_4 ; GCN1-NEXT: .LBB69_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB69_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB69_2 ; GCN1-NEXT: .LBB69_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_or_b32_e32 v2, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_or_b32_e32 v3, v1, v3 ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB69_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB69_4 ; GCN2-NEXT: .LBB69_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB69_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB69_2 ; GCN2-NEXT: .LBB69_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_or_b32_e32 v2, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_or_b32_e32 v3, v1, v3 ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB69_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB69_4 ; GCN3-NEXT: .LBB69_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB69_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB69_2 ; GCN3-NEXT: .LBB69_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_or_b32_e32 v3, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_or_b32_e32 v2, v0, v2 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } ; --------------------------------------------------------------------- ; atomicrmw xor ; --------------------------------------------------------------------- define void @flat_atomic_xor_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_xor_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB70_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB70_6 ; GCN1-NEXT: .LBB70_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB70_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB70_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v5, v7, v3 ; GCN1-NEXT: v_xor_b32_e32 v4, v6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB70_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB70_2 ; GCN1-NEXT: .LBB70_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xor_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB70_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB70_6 ; GCN2-NEXT: .LBB70_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB70_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB70_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v5, v7, v3 ; GCN2-NEXT: v_xor_b32_e32 v4, v6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB70_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB70_2 ; GCN2-NEXT: .LBB70_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xor_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB70_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB70_6 ; GCN3-NEXT: .LBB70_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB70_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB70_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v5, v7, v3 ; GCN3-NEXT: v_xor_b32_e32 v4, v6, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB70_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB70_2 ; GCN3-NEXT: .LBB70_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_xor_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_xor_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB71_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB71_6 ; GCN1-NEXT: .LBB71_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB71_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB71_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v5, v7, v3 ; GCN1-NEXT: v_xor_b32_e32 v4, v6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB71_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB71_2 ; GCN1-NEXT: .LBB71_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xor_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB71_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB71_6 ; GCN2-NEXT: .LBB71_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB71_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB71_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v5, v7, v3 ; GCN2-NEXT: v_xor_b32_e32 v4, v6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB71_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB71_2 ; GCN2-NEXT: .LBB71_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xor_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB71_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB71_6 ; GCN3-NEXT: .LBB71_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB71_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB71_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v5, v7, v3 ; GCN3-NEXT: v_xor_b32_e32 v4, v6, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB71_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB71_2 ; GCN3-NEXT: .LBB71_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_xor_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_xor_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB72_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v5, v[4:5] ; GCN1-NEXT: flat_load_dword v4, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB72_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: v_xor_b32_e32 v5, v7, v3 ; GCN1-NEXT: v_xor_b32_e32 v4, v6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB72_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: .LBB72_4: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB72_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: .LBB72_6: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v0, v4 ; GCN1-NEXT: v_mov_b32_e32 v1, v5 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xor_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB72_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v5, v[4:5] ; GCN2-NEXT: flat_load_dword v4, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB72_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: v_xor_b32_e32 v5, v7, v3 ; GCN2-NEXT: v_xor_b32_e32 v4, v6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB72_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: .LBB72_4: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB72_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: .LBB72_6: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v0, v4 ; GCN2-NEXT: v_mov_b32_e32 v1, v5 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xor_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB72_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB72_6 ; GCN3-NEXT: .LBB72_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB72_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB72_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_xor_b32_e32 v7, v9, v3 ; GCN3-NEXT: v_xor_b32_e32 v6, v8, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB72_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB72_2 ; GCN3-NEXT: .LBB72_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw xor ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_xor_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_xor_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB73_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB73_6 ; GCN1-NEXT: .LBB73_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB73_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[4:5] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB73_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_xor_b32_e32 v7, v9, v3 ; GCN1-NEXT: v_xor_b32_e32 v6, v8, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB73_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: ; implicit-def: $vgpr2 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB73_2 ; GCN1-NEXT: .LBB73_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3 ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xor_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB73_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB73_6 ; GCN2-NEXT: .LBB73_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB73_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[4:5] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB73_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_xor_b32_e32 v7, v9, v3 ; GCN2-NEXT: v_xor_b32_e32 v6, v8, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB73_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: ; implicit-def: $vgpr2 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB73_2 ; GCN2-NEXT: .LBB73_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3 ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xor_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB73_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB73_6 ; GCN3-NEXT: .LBB73_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB73_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB73_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_xor_b32_e32 v7, v9, v3 ; GCN3-NEXT: v_xor_b32_e32 v6, v8, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB73_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: ; implicit-def: $vgpr2 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB73_2 ; GCN3-NEXT: .LBB73_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw xor ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_xor_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB74_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccnz .LBB74_6 ; GCN1-NEXT: .LBB74_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB74_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v4, s4 ; GCN1-NEXT: v_mov_b32_e32 v5, s5 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: .LBB74_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v1, s7, v3 ; GCN1-NEXT: v_xor_b32_e32 v0, s6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB74_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB74_2 ; GCN1-NEXT: .LBB74_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_xor_b32_e32 v2, s6, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v3, s7, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xor_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB74_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccnz .LBB74_6 ; GCN2-NEXT: .LBB74_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB74_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v4, s4 ; GCN2-NEXT: v_mov_b32_e32 v5, s5 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: .LBB74_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v1, s7, v3 ; GCN2-NEXT: v_xor_b32_e32 v0, s6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB74_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB74_2 ; GCN2-NEXT: .LBB74_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_xor_b32_e32 v2, s6, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v3, s7, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xor_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB74_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccnz .LBB74_6 ; GCN3-NEXT: .LBB74_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB74_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s4 ; GCN3-NEXT: v_mov_b32_e32 v5, s5 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: .LBB74_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v1, s7, v3 ; GCN3-NEXT: v_xor_b32_e32 v0, s6, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB74_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB74_2 ; GCN3-NEXT: .LBB74_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_xor_b32_e32 v1, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v2, s6, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_xor_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB75_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccnz .LBB75_6 ; GCN1-NEXT: .LBB75_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB75_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v4, s34 ; GCN1-NEXT: v_mov_b32_e32 v5, s35 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: .LBB75_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v1, s7, v3 ; GCN1-NEXT: v_xor_b32_e32 v0, s6, v2 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB75_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB75_2 ; GCN1-NEXT: .LBB75_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v1, s34 ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_xor_b32_e32 v2, s6, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v3, s7, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xor_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB75_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccnz .LBB75_6 ; GCN2-NEXT: .LBB75_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB75_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v4, s34 ; GCN2-NEXT: v_mov_b32_e32 v5, s35 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: .LBB75_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v1, s7, v3 ; GCN2-NEXT: v_xor_b32_e32 v0, s6, v2 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB75_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB75_2 ; GCN2-NEXT: .LBB75_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v1, s34 ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_xor_b32_e32 v2, s6, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v3, s7, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xor_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB75_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccnz .LBB75_6 ; GCN3-NEXT: .LBB75_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB75_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s34 ; GCN3-NEXT: v_mov_b32_e32 v5, s35 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: .LBB75_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v1, s7, v3 ; GCN3-NEXT: v_xor_b32_e32 v0, s6, v2 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB75_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB75_2 ; GCN3-NEXT: .LBB75_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v0, s34 ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_xor_b32_e32 v1, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v2, s6, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_xor_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB76_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v2, s4 ; GCN1-NEXT: v_mov_b32_e32 v3, s5 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: .LBB76_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v1 ; GCN1-NEXT: v_mov_b32_e32 v6, v0 ; GCN1-NEXT: v_xor_b32_e32 v5, s7, v7 ; GCN1-NEXT: v_xor_b32_e32 v4, s6, v6 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB76_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB76_6 ; GCN1-NEXT: .LBB76_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB76_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_xor_b32_e32 v4, s6, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v5, s7, v1 ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB76_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xor_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB76_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v2, s4 ; GCN2-NEXT: v_mov_b32_e32 v3, s5 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: .LBB76_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v1 ; GCN2-NEXT: v_mov_b32_e32 v6, v0 ; GCN2-NEXT: v_xor_b32_e32 v5, s7, v7 ; GCN2-NEXT: v_xor_b32_e32 v4, s6, v6 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB76_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB76_6 ; GCN2-NEXT: .LBB76_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB76_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_xor_b32_e32 v4, s6, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v5, s7, v1 ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB76_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xor_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB76_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s4 ; GCN3-NEXT: v_mov_b32_e32 v3, s5 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: .LBB76_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v7, v1 ; GCN3-NEXT: v_mov_b32_e32 v6, v0 ; GCN3-NEXT: v_xor_b32_e32 v5, s7, v7 ; GCN3-NEXT: v_xor_b32_e32 v4, s6, v6 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB76_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB76_6 ; GCN3-NEXT: .LBB76_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB76_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_xor_b32_e32 v3, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v4, s6, v0 ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB76_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw xor ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_xor_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB77_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: v_mov_b32_e32 v3, s35 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: .LBB77_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v1 ; GCN1-NEXT: v_mov_b32_e32 v6, v0 ; GCN1-NEXT: v_xor_b32_e32 v5, s7, v7 ; GCN1-NEXT: v_xor_b32_e32 v4, s6, v6 ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB77_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB77_6 ; GCN1-NEXT: .LBB77_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB77_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_xor_b32_e32 v4, s6, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v5, s7, v1 ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB77_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xor_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB77_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: v_mov_b32_e32 v3, s35 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: .LBB77_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v1 ; GCN2-NEXT: v_mov_b32_e32 v6, v0 ; GCN2-NEXT: v_xor_b32_e32 v5, s7, v7 ; GCN2-NEXT: v_xor_b32_e32 v4, s6, v6 ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB77_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB77_6 ; GCN2-NEXT: .LBB77_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB77_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_xor_b32_e32 v4, s6, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v5, s7, v1 ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB77_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xor_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB77_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: v_mov_b32_e32 v3, s35 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: .LBB77_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v7, v1 ; GCN3-NEXT: v_mov_b32_e32 v6, v0 ; GCN3-NEXT: v_xor_b32_e32 v5, s7, v7 ; GCN3-NEXT: v_xor_b32_e32 v4, s6, v6 ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB77_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB77_6 ; GCN3-NEXT: .LBB77_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB77_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_xor_b32_e32 v3, s7, v1 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v4, s6, v0 ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB77_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw xor ptr %gep, i64 %in seq_cst ret i64 %result } define void @flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB78_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB78_4 ; GCN1-NEXT: .LBB78_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB78_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB78_2 ; GCN1-NEXT: .LBB78_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3 ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB78_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB78_4 ; GCN2-NEXT: .LBB78_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB78_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB78_2 ; GCN2-NEXT: .LBB78_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3 ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB78_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB78_4 ; GCN3-NEXT: .LBB78_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB78_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB78_2 ; GCN3-NEXT: .LBB78_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2 ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB79_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB79_4 ; GCN1-NEXT: .LBB79_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB79_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB79_2 ; GCN1-NEXT: .LBB79_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3 ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB79_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB79_4 ; GCN2-NEXT: .LBB79_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB79_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB79_2 ; GCN2-NEXT: .LBB79_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3 ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB79_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB79_4 ; GCN3-NEXT: .LBB79_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB79_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB79_2 ; GCN3-NEXT: .LBB79_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2 ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } ; --------------------------------------------------------------------- ; atomicrmw max ; --------------------------------------------------------------------- define void @flat_atomic_max_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_max_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB80_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB80_6 ; GCN1-NEXT: .LBB80_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB80_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB80_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB80_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB80_2 ; GCN1-NEXT: .LBB80_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_max_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB80_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB80_6 ; GCN2-NEXT: .LBB80_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB80_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB80_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB80_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB80_2 ; GCN2-NEXT: .LBB80_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_max_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB80_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB80_6 ; GCN3-NEXT: .LBB80_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB80_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB80_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB80_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB80_2 ; GCN3-NEXT: .LBB80_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_max_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_max_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB81_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB81_6 ; GCN1-NEXT: .LBB81_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB81_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB81_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB81_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB81_2 ; GCN1-NEXT: .LBB81_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_max_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB81_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB81_6 ; GCN2-NEXT: .LBB81_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB81_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB81_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB81_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB81_2 ; GCN2-NEXT: .LBB81_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_max_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB81_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB81_6 ; GCN3-NEXT: .LBB81_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB81_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB81_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB81_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB81_2 ; GCN3-NEXT: .LBB81_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_max_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_max_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB82_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v5, v[4:5] ; GCN1-NEXT: flat_load_dword v4, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB82_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB82_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: .LBB82_4: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB82_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: .LBB82_6: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v0, v4 ; GCN1-NEXT: v_mov_b32_e32 v1, v5 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_max_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB82_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v5, v[4:5] ; GCN2-NEXT: flat_load_dword v4, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB82_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB82_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: .LBB82_4: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB82_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: .LBB82_6: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v0, v4 ; GCN2-NEXT: v_mov_b32_e32 v1, v5 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_max_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB82_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB82_6 ; GCN3-NEXT: .LBB82_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB82_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB82_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB82_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB82_2 ; GCN3-NEXT: .LBB82_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw max ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_max_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_max_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB83_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB83_6 ; GCN1-NEXT: .LBB83_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB83_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[4:5] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB83_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB83_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB83_2 ; GCN1-NEXT: .LBB83_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_max_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB83_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB83_6 ; GCN2-NEXT: .LBB83_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB83_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[4:5] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB83_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB83_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB83_2 ; GCN2-NEXT: .LBB83_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_max_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB83_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB83_6 ; GCN3-NEXT: .LBB83_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB83_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB83_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB83_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB83_2 ; GCN3-NEXT: .LBB83_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw max ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_max_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB84_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccnz .LBB84_6 ; GCN1-NEXT: .LBB84_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB84_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v4, s4 ; GCN1-NEXT: v_mov_b32_e32 v5, s5 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s7 ; GCN1-NEXT: v_mov_b32_e32 v7, s6 ; GCN1-NEXT: .LBB84_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB84_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB84_2 ; GCN1-NEXT: .LBB84_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_max_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB84_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccnz .LBB84_6 ; GCN2-NEXT: .LBB84_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB84_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v4, s4 ; GCN2-NEXT: v_mov_b32_e32 v5, s5 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s7 ; GCN2-NEXT: v_mov_b32_e32 v7, s6 ; GCN2-NEXT: .LBB84_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB84_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB84_2 ; GCN2-NEXT: .LBB84_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_max_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB84_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccnz .LBB84_6 ; GCN3-NEXT: .LBB84_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB84_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s4 ; GCN3-NEXT: v_mov_b32_e32 v5, s5 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s7 ; GCN3-NEXT: v_mov_b32_e32 v7, s6 ; GCN3-NEXT: .LBB84_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB84_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB84_2 ; GCN3-NEXT: .LBB84_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_max_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB85_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccnz .LBB85_6 ; GCN1-NEXT: .LBB85_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB85_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v4, s34 ; GCN1-NEXT: v_mov_b32_e32 v5, s35 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s7 ; GCN1-NEXT: v_mov_b32_e32 v7, s6 ; GCN1-NEXT: .LBB85_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB85_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB85_2 ; GCN1-NEXT: .LBB85_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_max_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB85_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccnz .LBB85_6 ; GCN2-NEXT: .LBB85_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB85_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v4, s34 ; GCN2-NEXT: v_mov_b32_e32 v5, s35 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s7 ; GCN2-NEXT: v_mov_b32_e32 v7, s6 ; GCN2-NEXT: .LBB85_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB85_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB85_2 ; GCN2-NEXT: .LBB85_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_max_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB85_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccnz .LBB85_6 ; GCN3-NEXT: .LBB85_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB85_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s34 ; GCN3-NEXT: v_mov_b32_e32 v5, s35 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s7 ; GCN3-NEXT: v_mov_b32_e32 v7, s6 ; GCN3-NEXT: .LBB85_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB85_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB85_2 ; GCN3-NEXT: .LBB85_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_max_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB86_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v2, s4 ; GCN1-NEXT: v_mov_b32_e32 v3, s5 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: .LBB86_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB86_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB86_6 ; GCN1-NEXT: .LBB86_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB86_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB86_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_max_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB86_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v2, s4 ; GCN2-NEXT: v_mov_b32_e32 v3, s5 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: .LBB86_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB86_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB86_6 ; GCN2-NEXT: .LBB86_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB86_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB86_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_max_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB86_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s4 ; GCN3-NEXT: v_mov_b32_e32 v3, s5 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s7 ; GCN3-NEXT: v_mov_b32_e32 v5, s6 ; GCN3-NEXT: .LBB86_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB86_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB86_6 ; GCN3-NEXT: .LBB86_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB86_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB86_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw max ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_max_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB87_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: v_mov_b32_e32 v3, s35 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: .LBB87_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB87_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB87_6 ; GCN1-NEXT: .LBB87_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB87_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB87_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_max_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB87_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: v_mov_b32_e32 v3, s35 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: .LBB87_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB87_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB87_6 ; GCN2-NEXT: .LBB87_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB87_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB87_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_max_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB87_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: v_mov_b32_e32 v3, s35 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s7 ; GCN3-NEXT: v_mov_b32_e32 v5, s6 ; GCN3-NEXT: .LBB87_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB87_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB87_6 ; GCN3-NEXT: .LBB87_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB87_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB87_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw max ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %index) { ; GCN1-LABEL: atomic_max_i64_addr64_offset: ; GCN1: ; %bb.0: ; %entry ; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GCN1-NEXT: s_mov_b32 s14, -1 ; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd ; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f ; GCN1-NEXT: s_mov_b32 s15, 0xe8f000 ; GCN1-NEXT: s_add_u32 s12, s12, s11 ; GCN1-NEXT: s_addc_u32 s13, s13, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 ; GCN1-NEXT: s_add_u32 s0, s0, s4 ; GCN1-NEXT: s_addc_u32 s1, s1, s5 ; GCN1-NEXT: s_add_u32 s0, s0, 32 ; GCN1-NEXT: s_addc_u32 s1, s1, 0 ; GCN1-NEXT: s_cmp_eq_u32 s1, s8 ; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN1-NEXT: s_mov_b64 s[4:5], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB88_3 ; GCN1-NEXT: ; %bb.1: ; %Flow6 ; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN1-NEXT: s_cbranch_vccnz .LBB88_6 ; GCN1-NEXT: .LBB88_2: ; %atomicrmw.phi ; GCN1-NEXT: s_endpgm ; GCN1-NEXT: .LBB88_3: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v5, s1 ; GCN1-NEXT: v_mov_b32_e32 v4, s0 ; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN1-NEXT: s_mov_b64 s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s3 ; GCN1-NEXT: v_mov_b32_e32 v7, s2 ; GCN1-NEXT: .LBB88_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB88_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_branch .LBB88_2 ; GCN1-NEXT: .LBB88_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s2 ; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: s_add_i32 s0, s0, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s0 ; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s3 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen ; GCN1-NEXT: s_endpgm ; ; GCN2-LABEL: atomic_max_i64_addr64_offset: ; GCN2: ; %bb.0: ; %entry ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 ; GCN2-NEXT: s_mov_b32 s90, -1 ; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 ; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 ; GCN2-NEXT: s_add_u32 s88, s88, s11 ; GCN2-NEXT: s_addc_u32 s89, s89, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 ; GCN2-NEXT: s_add_u32 s0, s0, s4 ; GCN2-NEXT: s_addc_u32 s1, s1, s5 ; GCN2-NEXT: s_add_u32 s0, s0, 32 ; GCN2-NEXT: s_addc_u32 s1, s1, 0 ; GCN2-NEXT: s_cmp_eq_u32 s1, s8 ; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN2-NEXT: s_mov_b64 s[4:5], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB88_3 ; GCN2-NEXT: ; %bb.1: ; %Flow6 ; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN2-NEXT: s_cbranch_vccnz .LBB88_6 ; GCN2-NEXT: .LBB88_2: ; %atomicrmw.phi ; GCN2-NEXT: s_endpgm ; GCN2-NEXT: .LBB88_3: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v5, s1 ; GCN2-NEXT: v_mov_b32_e32 v4, s0 ; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN2-NEXT: s_mov_b64 s[4:5], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s3 ; GCN2-NEXT: v_mov_b32_e32 v7, s2 ; GCN2-NEXT: .LBB88_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB88_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_branch .LBB88_2 ; GCN2-NEXT: .LBB88_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: s_add_i32 s0, s0, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s0 ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s2 ; GCN2-NEXT: v_mov_b32_e32 v4, s3 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: s_endpgm ; ; GCN3-LABEL: atomic_max_i64_addr64_offset: ; GCN3: ; %bb.0: ; %entry ; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GCN3-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34 ; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GCN3-NEXT: s_mov_b32 s14, -1 ; GCN3-NEXT: s_mov_b32 s15, 0xe00000 ; GCN3-NEXT: s_add_u32 s12, s12, s11 ; GCN3-NEXT: s_addc_u32 s13, s13, 0 ; GCN3-NEXT: s_waitcnt lgkmcnt(0) ; GCN3-NEXT: s_lshl_b64 s[4:5], s[8:9], 3 ; GCN3-NEXT: s_add_u32 s0, s0, s4 ; GCN3-NEXT: s_addc_u32 s1, s1, s5 ; GCN3-NEXT: s_add_u32 s0, s0, 32 ; GCN3-NEXT: s_mov_b64 s[6:7], src_private_base ; GCN3-NEXT: s_addc_u32 s1, s1, 0 ; GCN3-NEXT: s_cmp_eq_u32 s1, s7 ; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN3-NEXT: s_mov_b64 s[4:5], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB88_3 ; GCN3-NEXT: ; %bb.1: ; %Flow6 ; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN3-NEXT: s_cbranch_vccnz .LBB88_6 ; GCN3-NEXT: .LBB88_2: ; %atomicrmw.phi ; GCN3-NEXT: s_endpgm ; GCN3-NEXT: .LBB88_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v5, s1 ; GCN3-NEXT: v_mov_b32_e32 v4, s0 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[4:5], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s3 ; GCN3-NEXT: v_mov_b32_e32 v7, s2 ; GCN3-NEXT: .LBB88_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB88_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_branch .LBB88_2 ; GCN3-NEXT: .LBB88_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s2 ; GCN3-NEXT: v_mov_b32_e32 v3, s3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4 ; GCN3-NEXT: s_endpgm entry: %ptr = getelementptr i64, ptr %out, i64 %index %gep = getelementptr i64, ptr %ptr, i64 4 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst ret void } define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) { ; GCN1-LABEL: atomic_max_i64_ret_addr64_offset: ; GCN1: ; %bb.0: ; %entry ; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GCN1-NEXT: s_mov_b32 s18, -1 ; GCN1-NEXT: s_mov_b32 s19, 0xe8f000 ; GCN1-NEXT: s_add_u32 s16, s16, s11 ; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9 ; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41 ; GCN1-NEXT: s_addc_u32 s17, s17, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN1-NEXT: s_add_u32 s0, s8, s0 ; GCN1-NEXT: s_addc_u32 s1, s9, s1 ; GCN1-NEXT: s_add_u32 s0, s0, 32 ; GCN1-NEXT: s_addc_u32 s1, s1, 0 ; GCN1-NEXT: s_cmp_eq_u32 s1, s2 ; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN1-NEXT: s_cbranch_vccz .LBB89_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v3, s1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN1-NEXT: s_mov_b64 s[2:3], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s13 ; GCN1-NEXT: v_mov_b32_e32 v5, s12 ; GCN1-NEXT: .LBB89_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN1-NEXT: s_cbranch_execnz .LBB89_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN1-NEXT: s_branch .LBB89_6 ; GCN1-NEXT: .LBB89_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB89_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s12 ; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: s_add_i32 s0, s0, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s0 ; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s13 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen ; GCN1-NEXT: .LBB89_6: ; %atomicrmw.phi ; GCN1-NEXT: v_mov_b32_e32 v2, s10 ; GCN1-NEXT: v_mov_b32_e32 v3, s11 ; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN1-NEXT: s_endpgm ; ; GCN2-LABEL: atomic_max_i64_ret_addr64_offset: ; GCN2: ; %bb.0: ; %entry ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 ; GCN2-NEXT: s_mov_b32 s90, -1 ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 ; GCN2-NEXT: s_add_u32 s88, s88, s11 ; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104 ; GCN2-NEXT: s_addc_u32 s89, s89, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN2-NEXT: s_add_u32 s0, s8, s0 ; GCN2-NEXT: s_addc_u32 s1, s9, s1 ; GCN2-NEXT: s_add_u32 s0, s0, 32 ; GCN2-NEXT: s_addc_u32 s1, s1, 0 ; GCN2-NEXT: s_cmp_eq_u32 s1, s2 ; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN2-NEXT: s_cbranch_vccz .LBB89_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v3, s1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN2-NEXT: s_mov_b64 s[2:3], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s13 ; GCN2-NEXT: v_mov_b32_e32 v5, s12 ; GCN2-NEXT: .LBB89_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN2-NEXT: s_cbranch_execnz .LBB89_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN2-NEXT: s_branch .LBB89_6 ; GCN2-NEXT: .LBB89_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB89_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: s_add_i32 s0, s0, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s0 ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s12 ; GCN2-NEXT: v_mov_b32_e32 v4, s13 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen ; GCN2-NEXT: .LBB89_6: ; %atomicrmw.phi ; GCN2-NEXT: v_mov_b32_e32 v2, s10 ; GCN2-NEXT: v_mov_b32_e32 v3, s11 ; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN2-NEXT: s_endpgm ; ; GCN3-LABEL: atomic_max_i64_ret_addr64_offset: ; GCN3: ; %bb.0: ; %entry ; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GCN3-NEXT: s_mov_b32 s18, -1 ; GCN3-NEXT: s_mov_b32 s19, 0xe00000 ; GCN3-NEXT: s_add_u32 s16, s16, s11 ; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GCN3-NEXT: s_addc_u32 s17, s17, 0 ; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base ; GCN3-NEXT: s_waitcnt lgkmcnt(0) ; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN3-NEXT: s_add_u32 s0, s8, s0 ; GCN3-NEXT: s_addc_u32 s1, s9, s1 ; GCN3-NEXT: s_add_u32 s0, s0, 32 ; GCN3-NEXT: s_addc_u32 s1, s1, 0 ; GCN3-NEXT: s_cmp_eq_u32 s1, s3 ; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN3-NEXT: s_cbranch_vccz .LBB89_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v3, s1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[2:3], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s13 ; GCN3-NEXT: v_mov_b32_e32 v5, s12 ; GCN3-NEXT: .LBB89_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN3-NEXT: s_cbranch_execnz .LBB89_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN3-NEXT: s_branch .LBB89_6 ; GCN3-NEXT: .LBB89_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB89_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s12 ; GCN3-NEXT: v_mov_b32_e32 v3, s13 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4 ; GCN3-NEXT: .LBB89_6: ; %atomicrmw.phi ; GCN3-NEXT: v_mov_b32_e32 v2, s10 ; GCN3-NEXT: v_mov_b32_e32 v3, s11 ; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN3-NEXT: s_endpgm entry: %ptr = getelementptr i64, ptr %out, i64 %index %gep = getelementptr i64, ptr %ptr, i64 4 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst store i64 %tmp0, ptr %out2 ret void } define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index) { ; GCN1-LABEL: atomic_max_i64_addr64: ; GCN1: ; %bb.0: ; %entry ; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GCN1-NEXT: s_mov_b32 s14, -1 ; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd ; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f ; GCN1-NEXT: s_mov_b32 s15, 0xe8f000 ; GCN1-NEXT: s_add_u32 s12, s12, s11 ; GCN1-NEXT: s_addc_u32 s13, s13, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 ; GCN1-NEXT: s_add_u32 s0, s0, s4 ; GCN1-NEXT: s_addc_u32 s1, s1, s5 ; GCN1-NEXT: s_cmp_eq_u32 s1, s8 ; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN1-NEXT: s_mov_b64 s[4:5], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB90_3 ; GCN1-NEXT: ; %bb.1: ; %Flow6 ; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN1-NEXT: s_cbranch_vccnz .LBB90_6 ; GCN1-NEXT: .LBB90_2: ; %atomicrmw.phi ; GCN1-NEXT: s_endpgm ; GCN1-NEXT: .LBB90_3: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v5, s1 ; GCN1-NEXT: v_mov_b32_e32 v4, s0 ; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN1-NEXT: s_mov_b64 s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s3 ; GCN1-NEXT: v_mov_b32_e32 v7, s2 ; GCN1-NEXT: .LBB90_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB90_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_branch .LBB90_2 ; GCN1-NEXT: .LBB90_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s2 ; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: s_add_i32 s0, s0, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s0 ; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s3 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen ; GCN1-NEXT: s_endpgm ; ; GCN2-LABEL: atomic_max_i64_addr64: ; GCN2: ; %bb.0: ; %entry ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 ; GCN2-NEXT: s_mov_b32 s90, -1 ; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 ; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 ; GCN2-NEXT: s_add_u32 s88, s88, s11 ; GCN2-NEXT: s_addc_u32 s89, s89, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 ; GCN2-NEXT: s_add_u32 s0, s0, s4 ; GCN2-NEXT: s_addc_u32 s1, s1, s5 ; GCN2-NEXT: s_cmp_eq_u32 s1, s8 ; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN2-NEXT: s_mov_b64 s[4:5], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB90_3 ; GCN2-NEXT: ; %bb.1: ; %Flow6 ; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN2-NEXT: s_cbranch_vccnz .LBB90_6 ; GCN2-NEXT: .LBB90_2: ; %atomicrmw.phi ; GCN2-NEXT: s_endpgm ; GCN2-NEXT: .LBB90_3: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v5, s1 ; GCN2-NEXT: v_mov_b32_e32 v4, s0 ; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN2-NEXT: s_mov_b64 s[4:5], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s3 ; GCN2-NEXT: v_mov_b32_e32 v7, s2 ; GCN2-NEXT: .LBB90_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB90_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_branch .LBB90_2 ; GCN2-NEXT: .LBB90_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: s_add_i32 s0, s0, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s0 ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s2 ; GCN2-NEXT: v_mov_b32_e32 v4, s3 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: s_endpgm ; ; GCN3-LABEL: atomic_max_i64_addr64: ; GCN3: ; %bb.0: ; %entry ; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GCN3-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34 ; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GCN3-NEXT: s_mov_b32 s14, -1 ; GCN3-NEXT: s_mov_b32 s15, 0xe00000 ; GCN3-NEXT: s_add_u32 s12, s12, s11 ; GCN3-NEXT: s_addc_u32 s13, s13, 0 ; GCN3-NEXT: s_waitcnt lgkmcnt(0) ; GCN3-NEXT: s_lshl_b64 s[4:5], s[8:9], 3 ; GCN3-NEXT: s_add_u32 s0, s0, s4 ; GCN3-NEXT: s_mov_b64 s[6:7], src_private_base ; GCN3-NEXT: s_addc_u32 s1, s1, s5 ; GCN3-NEXT: s_cmp_eq_u32 s1, s7 ; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN3-NEXT: s_mov_b64 s[4:5], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB90_3 ; GCN3-NEXT: ; %bb.1: ; %Flow6 ; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN3-NEXT: s_cbranch_vccnz .LBB90_6 ; GCN3-NEXT: .LBB90_2: ; %atomicrmw.phi ; GCN3-NEXT: s_endpgm ; GCN3-NEXT: .LBB90_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v5, s1 ; GCN3-NEXT: v_mov_b32_e32 v4, s0 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[4:5], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s3 ; GCN3-NEXT: v_mov_b32_e32 v7, s2 ; GCN3-NEXT: .LBB90_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB90_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_branch .LBB90_2 ; GCN3-NEXT: .LBB90_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s2 ; GCN3-NEXT: v_mov_b32_e32 v3, s3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4 ; GCN3-NEXT: s_endpgm entry: %ptr = getelementptr i64, ptr %out, i64 %index %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst ret void } define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) { ; GCN1-LABEL: atomic_max_i64_ret_addr64: ; GCN1: ; %bb.0: ; %entry ; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GCN1-NEXT: s_mov_b32 s18, -1 ; GCN1-NEXT: s_mov_b32 s19, 0xe8f000 ; GCN1-NEXT: s_add_u32 s16, s16, s11 ; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9 ; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41 ; GCN1-NEXT: s_addc_u32 s17, s17, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN1-NEXT: s_add_u32 s0, s8, s0 ; GCN1-NEXT: s_addc_u32 s1, s9, s1 ; GCN1-NEXT: s_cmp_eq_u32 s1, s2 ; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN1-NEXT: s_cbranch_vccz .LBB91_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v3, s1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN1-NEXT: s_mov_b64 s[2:3], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s13 ; GCN1-NEXT: v_mov_b32_e32 v5, s12 ; GCN1-NEXT: .LBB91_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN1-NEXT: s_cbranch_execnz .LBB91_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN1-NEXT: s_branch .LBB91_6 ; GCN1-NEXT: .LBB91_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB91_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s12 ; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: s_add_i32 s0, s0, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s0 ; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s13 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen ; GCN1-NEXT: .LBB91_6: ; %atomicrmw.phi ; GCN1-NEXT: v_mov_b32_e32 v2, s10 ; GCN1-NEXT: v_mov_b32_e32 v3, s11 ; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN1-NEXT: s_endpgm ; ; GCN2-LABEL: atomic_max_i64_ret_addr64: ; GCN2: ; %bb.0: ; %entry ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 ; GCN2-NEXT: s_mov_b32 s90, -1 ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 ; GCN2-NEXT: s_add_u32 s88, s88, s11 ; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104 ; GCN2-NEXT: s_addc_u32 s89, s89, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN2-NEXT: s_add_u32 s0, s8, s0 ; GCN2-NEXT: s_addc_u32 s1, s9, s1 ; GCN2-NEXT: s_cmp_eq_u32 s1, s2 ; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN2-NEXT: s_cbranch_vccz .LBB91_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v3, s1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN2-NEXT: s_mov_b64 s[2:3], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s13 ; GCN2-NEXT: v_mov_b32_e32 v5, s12 ; GCN2-NEXT: .LBB91_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN2-NEXT: s_cbranch_execnz .LBB91_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN2-NEXT: s_branch .LBB91_6 ; GCN2-NEXT: .LBB91_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB91_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: s_add_i32 s0, s0, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s0 ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s12 ; GCN2-NEXT: v_mov_b32_e32 v4, s13 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen ; GCN2-NEXT: .LBB91_6: ; %atomicrmw.phi ; GCN2-NEXT: v_mov_b32_e32 v2, s10 ; GCN2-NEXT: v_mov_b32_e32 v3, s11 ; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN2-NEXT: s_endpgm ; ; GCN3-LABEL: atomic_max_i64_ret_addr64: ; GCN3: ; %bb.0: ; %entry ; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GCN3-NEXT: s_mov_b32 s18, -1 ; GCN3-NEXT: s_mov_b32 s19, 0xe00000 ; GCN3-NEXT: s_add_u32 s16, s16, s11 ; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GCN3-NEXT: s_addc_u32 s17, s17, 0 ; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base ; GCN3-NEXT: s_waitcnt lgkmcnt(0) ; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN3-NEXT: s_add_u32 s0, s8, s0 ; GCN3-NEXT: s_addc_u32 s1, s9, s1 ; GCN3-NEXT: s_cmp_eq_u32 s1, s3 ; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN3-NEXT: s_cbranch_vccz .LBB91_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v3, s1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[2:3], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s13 ; GCN3-NEXT: v_mov_b32_e32 v5, s12 ; GCN3-NEXT: .LBB91_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN3-NEXT: s_cbranch_execnz .LBB91_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN3-NEXT: s_branch .LBB91_6 ; GCN3-NEXT: .LBB91_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB91_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s12 ; GCN3-NEXT: v_mov_b32_e32 v3, s13 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4 ; GCN3-NEXT: .LBB91_6: ; %atomicrmw.phi ; GCN3-NEXT: v_mov_b32_e32 v2, s10 ; GCN3-NEXT: v_mov_b32_e32 v3, s11 ; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN3-NEXT: s_endpgm entry: %ptr = getelementptr i64, ptr %out, i64 %index %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst store i64 %tmp0, ptr %out2 ret void } define void @flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB92_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB92_4 ; GCN1-NEXT: .LBB92_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB92_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB92_2 ; GCN1-NEXT: .LBB92_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB92_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB92_4 ; GCN2-NEXT: .LBB92_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB92_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB92_2 ; GCN2-NEXT: .LBB92_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB92_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB92_4 ; GCN3-NEXT: .LBB92_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB92_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB92_2 ; GCN3-NEXT: .LBB92_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB93_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB93_4 ; GCN1-NEXT: .LBB93_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB93_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB93_2 ; GCN1-NEXT: .LBB93_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB93_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB93_4 ; GCN2-NEXT: .LBB93_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB93_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB93_2 ; GCN2-NEXT: .LBB93_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB93_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB93_4 ; GCN3-NEXT: .LBB93_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB93_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB93_2 ; GCN3-NEXT: .LBB93_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw max ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } ; --------------------------------------------------------------------- ; atomicrmw umax ; --------------------------------------------------------------------- define void @flat_atomic_umax_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_umax_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB94_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB94_6 ; GCN1-NEXT: .LBB94_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB94_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB94_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB94_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB94_2 ; GCN1-NEXT: .LBB94_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umax_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB94_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB94_6 ; GCN2-NEXT: .LBB94_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB94_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB94_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB94_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB94_2 ; GCN2-NEXT: .LBB94_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umax_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB94_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB94_6 ; GCN3-NEXT: .LBB94_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB94_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB94_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB94_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB94_2 ; GCN3-NEXT: .LBB94_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_umax_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_umax_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB95_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB95_6 ; GCN1-NEXT: .LBB95_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB95_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB95_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB95_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB95_2 ; GCN1-NEXT: .LBB95_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umax_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB95_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB95_6 ; GCN2-NEXT: .LBB95_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB95_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB95_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB95_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB95_2 ; GCN2-NEXT: .LBB95_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umax_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB95_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB95_6 ; GCN3-NEXT: .LBB95_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB95_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB95_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB95_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB95_2 ; GCN3-NEXT: .LBB95_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_umax_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_umax_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB96_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v5, v[4:5] ; GCN1-NEXT: flat_load_dword v4, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB96_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB96_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: .LBB96_4: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB96_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: .LBB96_6: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v0, v4 ; GCN1-NEXT: v_mov_b32_e32 v1, v5 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umax_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB96_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v5, v[4:5] ; GCN2-NEXT: flat_load_dword v4, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB96_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB96_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: .LBB96_4: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB96_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: .LBB96_6: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v0, v4 ; GCN2-NEXT: v_mov_b32_e32 v1, v5 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umax_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB96_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB96_6 ; GCN3-NEXT: .LBB96_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB96_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB96_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB96_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB96_2 ; GCN3-NEXT: .LBB96_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw umax ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_umax_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_umax_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB97_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB97_6 ; GCN1-NEXT: .LBB97_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB97_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[4:5] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB97_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB97_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB97_2 ; GCN1-NEXT: .LBB97_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umax_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB97_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB97_6 ; GCN2-NEXT: .LBB97_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB97_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[4:5] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB97_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB97_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB97_2 ; GCN2-NEXT: .LBB97_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umax_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB97_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB97_6 ; GCN3-NEXT: .LBB97_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB97_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB97_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB97_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB97_2 ; GCN3-NEXT: .LBB97_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw umax ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_umax_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB98_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccnz .LBB98_6 ; GCN1-NEXT: .LBB98_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB98_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v4, s4 ; GCN1-NEXT: v_mov_b32_e32 v5, s5 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s7 ; GCN1-NEXT: v_mov_b32_e32 v7, s6 ; GCN1-NEXT: .LBB98_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB98_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB98_2 ; GCN1-NEXT: .LBB98_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umax_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB98_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccnz .LBB98_6 ; GCN2-NEXT: .LBB98_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB98_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v4, s4 ; GCN2-NEXT: v_mov_b32_e32 v5, s5 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s7 ; GCN2-NEXT: v_mov_b32_e32 v7, s6 ; GCN2-NEXT: .LBB98_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB98_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB98_2 ; GCN2-NEXT: .LBB98_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umax_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB98_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccnz .LBB98_6 ; GCN3-NEXT: .LBB98_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB98_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s4 ; GCN3-NEXT: v_mov_b32_e32 v5, s5 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s7 ; GCN3-NEXT: v_mov_b32_e32 v7, s6 ; GCN3-NEXT: .LBB98_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB98_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB98_2 ; GCN3-NEXT: .LBB98_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_umax_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB99_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccnz .LBB99_6 ; GCN1-NEXT: .LBB99_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB99_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v4, s34 ; GCN1-NEXT: v_mov_b32_e32 v5, s35 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s7 ; GCN1-NEXT: v_mov_b32_e32 v7, s6 ; GCN1-NEXT: .LBB99_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB99_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB99_2 ; GCN1-NEXT: .LBB99_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umax_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB99_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccnz .LBB99_6 ; GCN2-NEXT: .LBB99_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB99_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v4, s34 ; GCN2-NEXT: v_mov_b32_e32 v5, s35 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s7 ; GCN2-NEXT: v_mov_b32_e32 v7, s6 ; GCN2-NEXT: .LBB99_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB99_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB99_2 ; GCN2-NEXT: .LBB99_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umax_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB99_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccnz .LBB99_6 ; GCN3-NEXT: .LBB99_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB99_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s34 ; GCN3-NEXT: v_mov_b32_e32 v5, s35 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s7 ; GCN3-NEXT: v_mov_b32_e32 v7, s6 ; GCN3-NEXT: .LBB99_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB99_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB99_2 ; GCN3-NEXT: .LBB99_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_umax_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB100_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v2, s4 ; GCN1-NEXT: v_mov_b32_e32 v3, s5 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: .LBB100_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB100_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB100_6 ; GCN1-NEXT: .LBB100_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB100_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB100_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umax_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB100_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v2, s4 ; GCN2-NEXT: v_mov_b32_e32 v3, s5 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: .LBB100_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB100_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB100_6 ; GCN2-NEXT: .LBB100_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB100_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB100_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umax_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB100_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s4 ; GCN3-NEXT: v_mov_b32_e32 v3, s5 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s7 ; GCN3-NEXT: v_mov_b32_e32 v5, s6 ; GCN3-NEXT: .LBB100_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB100_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB100_6 ; GCN3-NEXT: .LBB100_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB100_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB100_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw umax ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_umax_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB101_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: v_mov_b32_e32 v3, s35 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: .LBB101_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB101_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB101_6 ; GCN1-NEXT: .LBB101_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB101_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB101_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umax_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB101_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: v_mov_b32_e32 v3, s35 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: .LBB101_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB101_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB101_6 ; GCN2-NEXT: .LBB101_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB101_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB101_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umax_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB101_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: v_mov_b32_e32 v3, s35 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s7 ; GCN3-NEXT: v_mov_b32_e32 v5, s6 ; GCN3-NEXT: .LBB101_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB101_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB101_6 ; GCN3-NEXT: .LBB101_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB101_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB101_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw umax ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64 %index) { ; GCN1-LABEL: atomic_umax_i64_addr64_offset: ; GCN1: ; %bb.0: ; %entry ; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GCN1-NEXT: s_mov_b32 s14, -1 ; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd ; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f ; GCN1-NEXT: s_mov_b32 s15, 0xe8f000 ; GCN1-NEXT: s_add_u32 s12, s12, s11 ; GCN1-NEXT: s_addc_u32 s13, s13, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 ; GCN1-NEXT: s_add_u32 s0, s0, s4 ; GCN1-NEXT: s_addc_u32 s1, s1, s5 ; GCN1-NEXT: s_add_u32 s0, s0, 32 ; GCN1-NEXT: s_addc_u32 s1, s1, 0 ; GCN1-NEXT: s_cmp_eq_u32 s1, s8 ; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN1-NEXT: s_mov_b64 s[4:5], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB102_3 ; GCN1-NEXT: ; %bb.1: ; %Flow6 ; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN1-NEXT: s_cbranch_vccnz .LBB102_6 ; GCN1-NEXT: .LBB102_2: ; %atomicrmw.phi ; GCN1-NEXT: s_endpgm ; GCN1-NEXT: .LBB102_3: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v5, s1 ; GCN1-NEXT: v_mov_b32_e32 v4, s0 ; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN1-NEXT: s_mov_b64 s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s3 ; GCN1-NEXT: v_mov_b32_e32 v7, s2 ; GCN1-NEXT: .LBB102_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB102_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_branch .LBB102_2 ; GCN1-NEXT: .LBB102_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s2 ; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: s_add_i32 s0, s0, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s0 ; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s3 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen ; GCN1-NEXT: s_endpgm ; ; GCN2-LABEL: atomic_umax_i64_addr64_offset: ; GCN2: ; %bb.0: ; %entry ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 ; GCN2-NEXT: s_mov_b32 s90, -1 ; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 ; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 ; GCN2-NEXT: s_add_u32 s88, s88, s11 ; GCN2-NEXT: s_addc_u32 s89, s89, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 ; GCN2-NEXT: s_add_u32 s0, s0, s4 ; GCN2-NEXT: s_addc_u32 s1, s1, s5 ; GCN2-NEXT: s_add_u32 s0, s0, 32 ; GCN2-NEXT: s_addc_u32 s1, s1, 0 ; GCN2-NEXT: s_cmp_eq_u32 s1, s8 ; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN2-NEXT: s_mov_b64 s[4:5], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB102_3 ; GCN2-NEXT: ; %bb.1: ; %Flow6 ; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN2-NEXT: s_cbranch_vccnz .LBB102_6 ; GCN2-NEXT: .LBB102_2: ; %atomicrmw.phi ; GCN2-NEXT: s_endpgm ; GCN2-NEXT: .LBB102_3: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v5, s1 ; GCN2-NEXT: v_mov_b32_e32 v4, s0 ; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN2-NEXT: s_mov_b64 s[4:5], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s3 ; GCN2-NEXT: v_mov_b32_e32 v7, s2 ; GCN2-NEXT: .LBB102_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB102_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_branch .LBB102_2 ; GCN2-NEXT: .LBB102_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: s_add_i32 s0, s0, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s0 ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s2 ; GCN2-NEXT: v_mov_b32_e32 v4, s3 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: s_endpgm ; ; GCN3-LABEL: atomic_umax_i64_addr64_offset: ; GCN3: ; %bb.0: ; %entry ; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GCN3-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34 ; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GCN3-NEXT: s_mov_b32 s14, -1 ; GCN3-NEXT: s_mov_b32 s15, 0xe00000 ; GCN3-NEXT: s_add_u32 s12, s12, s11 ; GCN3-NEXT: s_addc_u32 s13, s13, 0 ; GCN3-NEXT: s_waitcnt lgkmcnt(0) ; GCN3-NEXT: s_lshl_b64 s[4:5], s[8:9], 3 ; GCN3-NEXT: s_add_u32 s0, s0, s4 ; GCN3-NEXT: s_addc_u32 s1, s1, s5 ; GCN3-NEXT: s_add_u32 s0, s0, 32 ; GCN3-NEXT: s_mov_b64 s[6:7], src_private_base ; GCN3-NEXT: s_addc_u32 s1, s1, 0 ; GCN3-NEXT: s_cmp_eq_u32 s1, s7 ; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN3-NEXT: s_mov_b64 s[4:5], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB102_3 ; GCN3-NEXT: ; %bb.1: ; %Flow6 ; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN3-NEXT: s_cbranch_vccnz .LBB102_6 ; GCN3-NEXT: .LBB102_2: ; %atomicrmw.phi ; GCN3-NEXT: s_endpgm ; GCN3-NEXT: .LBB102_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v5, s1 ; GCN3-NEXT: v_mov_b32_e32 v4, s0 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[4:5], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s3 ; GCN3-NEXT: v_mov_b32_e32 v7, s2 ; GCN3-NEXT: .LBB102_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB102_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_branch .LBB102_2 ; GCN3-NEXT: .LBB102_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s2 ; GCN3-NEXT: v_mov_b32_e32 v3, s3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4 ; GCN3-NEXT: s_endpgm entry: %ptr = getelementptr i64, ptr %out, i64 %index %gep = getelementptr i64, ptr %ptr, i64 4 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst ret void } define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) { ; GCN1-LABEL: atomic_umax_i64_ret_addr64_offset: ; GCN1: ; %bb.0: ; %entry ; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GCN1-NEXT: s_mov_b32 s18, -1 ; GCN1-NEXT: s_mov_b32 s19, 0xe8f000 ; GCN1-NEXT: s_add_u32 s16, s16, s11 ; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9 ; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41 ; GCN1-NEXT: s_addc_u32 s17, s17, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN1-NEXT: s_add_u32 s0, s8, s0 ; GCN1-NEXT: s_addc_u32 s1, s9, s1 ; GCN1-NEXT: s_add_u32 s0, s0, 32 ; GCN1-NEXT: s_addc_u32 s1, s1, 0 ; GCN1-NEXT: s_cmp_eq_u32 s1, s2 ; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN1-NEXT: s_cbranch_vccz .LBB103_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v3, s1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN1-NEXT: s_mov_b64 s[2:3], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s13 ; GCN1-NEXT: v_mov_b32_e32 v5, s12 ; GCN1-NEXT: .LBB103_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN1-NEXT: s_cbranch_execnz .LBB103_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN1-NEXT: s_branch .LBB103_6 ; GCN1-NEXT: .LBB103_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB103_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s12 ; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: s_add_i32 s0, s0, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s0 ; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s13 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen ; GCN1-NEXT: .LBB103_6: ; %atomicrmw.phi ; GCN1-NEXT: v_mov_b32_e32 v2, s10 ; GCN1-NEXT: v_mov_b32_e32 v3, s11 ; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN1-NEXT: s_endpgm ; ; GCN2-LABEL: atomic_umax_i64_ret_addr64_offset: ; GCN2: ; %bb.0: ; %entry ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 ; GCN2-NEXT: s_mov_b32 s90, -1 ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 ; GCN2-NEXT: s_add_u32 s88, s88, s11 ; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104 ; GCN2-NEXT: s_addc_u32 s89, s89, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN2-NEXT: s_add_u32 s0, s8, s0 ; GCN2-NEXT: s_addc_u32 s1, s9, s1 ; GCN2-NEXT: s_add_u32 s0, s0, 32 ; GCN2-NEXT: s_addc_u32 s1, s1, 0 ; GCN2-NEXT: s_cmp_eq_u32 s1, s2 ; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN2-NEXT: s_cbranch_vccz .LBB103_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v3, s1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN2-NEXT: s_mov_b64 s[2:3], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s13 ; GCN2-NEXT: v_mov_b32_e32 v5, s12 ; GCN2-NEXT: .LBB103_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN2-NEXT: s_cbranch_execnz .LBB103_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN2-NEXT: s_branch .LBB103_6 ; GCN2-NEXT: .LBB103_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB103_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: s_add_i32 s0, s0, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s0 ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s12 ; GCN2-NEXT: v_mov_b32_e32 v4, s13 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen ; GCN2-NEXT: .LBB103_6: ; %atomicrmw.phi ; GCN2-NEXT: v_mov_b32_e32 v2, s10 ; GCN2-NEXT: v_mov_b32_e32 v3, s11 ; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN2-NEXT: s_endpgm ; ; GCN3-LABEL: atomic_umax_i64_ret_addr64_offset: ; GCN3: ; %bb.0: ; %entry ; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GCN3-NEXT: s_mov_b32 s18, -1 ; GCN3-NEXT: s_mov_b32 s19, 0xe00000 ; GCN3-NEXT: s_add_u32 s16, s16, s11 ; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GCN3-NEXT: s_addc_u32 s17, s17, 0 ; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base ; GCN3-NEXT: s_waitcnt lgkmcnt(0) ; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN3-NEXT: s_add_u32 s0, s8, s0 ; GCN3-NEXT: s_addc_u32 s1, s9, s1 ; GCN3-NEXT: s_add_u32 s0, s0, 32 ; GCN3-NEXT: s_addc_u32 s1, s1, 0 ; GCN3-NEXT: s_cmp_eq_u32 s1, s3 ; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN3-NEXT: s_cbranch_vccz .LBB103_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v3, s1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[2:3], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s13 ; GCN3-NEXT: v_mov_b32_e32 v5, s12 ; GCN3-NEXT: .LBB103_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN3-NEXT: s_cbranch_execnz .LBB103_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN3-NEXT: s_branch .LBB103_6 ; GCN3-NEXT: .LBB103_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB103_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s12 ; GCN3-NEXT: v_mov_b32_e32 v3, s13 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4 ; GCN3-NEXT: .LBB103_6: ; %atomicrmw.phi ; GCN3-NEXT: v_mov_b32_e32 v2, s10 ; GCN3-NEXT: v_mov_b32_e32 v3, s11 ; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN3-NEXT: s_endpgm entry: %ptr = getelementptr i64, ptr %out, i64 %index %gep = getelementptr i64, ptr %ptr, i64 4 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst store i64 %tmp0, ptr %out2 ret void } define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) { ; GCN1-LABEL: atomic_umax_i64_ret_addr64: ; GCN1: ; %bb.0: ; %entry ; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GCN1-NEXT: s_mov_b32 s18, -1 ; GCN1-NEXT: s_mov_b32 s19, 0xe8f000 ; GCN1-NEXT: s_add_u32 s16, s16, s11 ; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9 ; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41 ; GCN1-NEXT: s_addc_u32 s17, s17, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN1-NEXT: s_add_u32 s0, s8, s0 ; GCN1-NEXT: s_addc_u32 s1, s9, s1 ; GCN1-NEXT: s_cmp_eq_u32 s1, s2 ; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN1-NEXT: s_cbranch_vccz .LBB104_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v3, s1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN1-NEXT: s_mov_b64 s[2:3], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s13 ; GCN1-NEXT: v_mov_b32_e32 v5, s12 ; GCN1-NEXT: .LBB104_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN1-NEXT: s_cbranch_execnz .LBB104_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN1-NEXT: s_branch .LBB104_6 ; GCN1-NEXT: .LBB104_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB104_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s12 ; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: s_add_i32 s0, s0, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s0 ; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s13 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen ; GCN1-NEXT: .LBB104_6: ; %atomicrmw.phi ; GCN1-NEXT: v_mov_b32_e32 v2, s10 ; GCN1-NEXT: v_mov_b32_e32 v3, s11 ; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN1-NEXT: s_endpgm ; ; GCN2-LABEL: atomic_umax_i64_ret_addr64: ; GCN2: ; %bb.0: ; %entry ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 ; GCN2-NEXT: s_mov_b32 s90, -1 ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 ; GCN2-NEXT: s_add_u32 s88, s88, s11 ; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104 ; GCN2-NEXT: s_addc_u32 s89, s89, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN2-NEXT: s_add_u32 s0, s8, s0 ; GCN2-NEXT: s_addc_u32 s1, s9, s1 ; GCN2-NEXT: s_cmp_eq_u32 s1, s2 ; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN2-NEXT: s_cbranch_vccz .LBB104_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v3, s1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN2-NEXT: s_mov_b64 s[2:3], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s13 ; GCN2-NEXT: v_mov_b32_e32 v5, s12 ; GCN2-NEXT: .LBB104_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN2-NEXT: s_cbranch_execnz .LBB104_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN2-NEXT: s_branch .LBB104_6 ; GCN2-NEXT: .LBB104_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB104_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: s_add_i32 s0, s0, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s0 ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s12 ; GCN2-NEXT: v_mov_b32_e32 v4, s13 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen ; GCN2-NEXT: .LBB104_6: ; %atomicrmw.phi ; GCN2-NEXT: v_mov_b32_e32 v2, s10 ; GCN2-NEXT: v_mov_b32_e32 v3, s11 ; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN2-NEXT: s_endpgm ; ; GCN3-LABEL: atomic_umax_i64_ret_addr64: ; GCN3: ; %bb.0: ; %entry ; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GCN3-NEXT: s_mov_b32 s18, -1 ; GCN3-NEXT: s_mov_b32 s19, 0xe00000 ; GCN3-NEXT: s_add_u32 s16, s16, s11 ; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GCN3-NEXT: s_addc_u32 s17, s17, 0 ; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base ; GCN3-NEXT: s_waitcnt lgkmcnt(0) ; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN3-NEXT: s_add_u32 s0, s8, s0 ; GCN3-NEXT: s_addc_u32 s1, s9, s1 ; GCN3-NEXT: s_cmp_eq_u32 s1, s3 ; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN3-NEXT: s_cbranch_vccz .LBB104_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v3, s1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[2:3], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s13 ; GCN3-NEXT: v_mov_b32_e32 v5, s12 ; GCN3-NEXT: .LBB104_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN3-NEXT: s_cbranch_execnz .LBB104_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN3-NEXT: s_branch .LBB104_6 ; GCN3-NEXT: .LBB104_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB104_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s12 ; GCN3-NEXT: v_mov_b32_e32 v3, s13 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4 ; GCN3-NEXT: .LBB104_6: ; %atomicrmw.phi ; GCN3-NEXT: v_mov_b32_e32 v2, s10 ; GCN3-NEXT: v_mov_b32_e32 v3, s11 ; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN3-NEXT: s_endpgm entry: %ptr = getelementptr i64, ptr %out, i64 %index %tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst store i64 %tmp0, ptr %out2 ret void } define void @flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB105_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB105_4 ; GCN1-NEXT: .LBB105_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB105_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB105_2 ; GCN1-NEXT: .LBB105_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB105_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB105_4 ; GCN2-NEXT: .LBB105_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB105_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB105_2 ; GCN2-NEXT: .LBB105_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB105_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB105_4 ; GCN3-NEXT: .LBB105_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB105_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB105_2 ; GCN3-NEXT: .LBB105_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB106_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB106_4 ; GCN1-NEXT: .LBB106_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB106_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB106_2 ; GCN1-NEXT: .LBB106_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB106_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB106_4 ; GCN2-NEXT: .LBB106_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB106_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB106_2 ; GCN2-NEXT: .LBB106_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB106_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB106_4 ; GCN3-NEXT: .LBB106_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB106_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB106_2 ; GCN3-NEXT: .LBB106_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw umax ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } ; --------------------------------------------------------------------- ; atomicrmw umin ; --------------------------------------------------------------------- define void @flat_atomic_umin_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_umin_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB107_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB107_6 ; GCN1-NEXT: .LBB107_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB107_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB107_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB107_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB107_2 ; GCN1-NEXT: .LBB107_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umin_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB107_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB107_6 ; GCN2-NEXT: .LBB107_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB107_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB107_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB107_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB107_2 ; GCN2-NEXT: .LBB107_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umin_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB107_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB107_6 ; GCN3-NEXT: .LBB107_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB107_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB107_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB107_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB107_2 ; GCN3-NEXT: .LBB107_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_umin_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_umin_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB108_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB108_6 ; GCN1-NEXT: .LBB108_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB108_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB108_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB108_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB108_2 ; GCN1-NEXT: .LBB108_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umin_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB108_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB108_6 ; GCN2-NEXT: .LBB108_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB108_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB108_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB108_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB108_2 ; GCN2-NEXT: .LBB108_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umin_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB108_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB108_6 ; GCN3-NEXT: .LBB108_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB108_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB108_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB108_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB108_2 ; GCN3-NEXT: .LBB108_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_umin_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_umin_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB109_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v5, v[4:5] ; GCN1-NEXT: flat_load_dword v4, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB109_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB109_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: .LBB109_4: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB109_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[4:5], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: .LBB109_6: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v0, v4 ; GCN1-NEXT: v_mov_b32_e32 v1, v5 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umin_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB109_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v5, v[4:5] ; GCN2-NEXT: flat_load_dword v4, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB109_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB109_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: .LBB109_4: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB109_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[4:5], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: .LBB109_6: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v0, v4 ; GCN2-NEXT: v_mov_b32_e32 v1, v5 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umin_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB109_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB109_6 ; GCN3-NEXT: .LBB109_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB109_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB109_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB109_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB109_2 ; GCN3-NEXT: .LBB109_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw umin ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_umin_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_umin_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB110_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB110_6 ; GCN1-NEXT: .LBB110_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB110_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[4:5] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB110_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB110_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB110_2 ; GCN1-NEXT: .LBB110_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umin_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB110_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB110_6 ; GCN2-NEXT: .LBB110_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB110_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[4:5] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB110_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB110_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB110_2 ; GCN2-NEXT: .LBB110_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umin_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB110_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB110_6 ; GCN3-NEXT: .LBB110_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB110_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB110_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB110_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB110_2 ; GCN3-NEXT: .LBB110_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw umin ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_umin_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB111_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccnz .LBB111_6 ; GCN1-NEXT: .LBB111_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB111_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v4, s4 ; GCN1-NEXT: v_mov_b32_e32 v5, s5 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s7 ; GCN1-NEXT: v_mov_b32_e32 v7, s6 ; GCN1-NEXT: .LBB111_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB111_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB111_2 ; GCN1-NEXT: .LBB111_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umin_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB111_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccnz .LBB111_6 ; GCN2-NEXT: .LBB111_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB111_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v4, s4 ; GCN2-NEXT: v_mov_b32_e32 v5, s5 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s7 ; GCN2-NEXT: v_mov_b32_e32 v7, s6 ; GCN2-NEXT: .LBB111_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB111_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB111_2 ; GCN2-NEXT: .LBB111_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umin_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB111_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccnz .LBB111_6 ; GCN3-NEXT: .LBB111_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB111_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s4 ; GCN3-NEXT: v_mov_b32_e32 v5, s5 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s7 ; GCN3-NEXT: v_mov_b32_e32 v7, s6 ; GCN3-NEXT: .LBB111_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB111_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB111_2 ; GCN3-NEXT: .LBB111_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_umin_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB112_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccnz .LBB112_6 ; GCN1-NEXT: .LBB112_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB112_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v4, s34 ; GCN1-NEXT: v_mov_b32_e32 v5, s35 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s7 ; GCN1-NEXT: v_mov_b32_e32 v7, s6 ; GCN1-NEXT: .LBB112_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB112_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB112_2 ; GCN1-NEXT: .LBB112_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umin_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB112_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccnz .LBB112_6 ; GCN2-NEXT: .LBB112_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB112_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v4, s34 ; GCN2-NEXT: v_mov_b32_e32 v5, s35 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s7 ; GCN2-NEXT: v_mov_b32_e32 v7, s6 ; GCN2-NEXT: .LBB112_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB112_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB112_2 ; GCN2-NEXT: .LBB112_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umin_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB112_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccnz .LBB112_6 ; GCN3-NEXT: .LBB112_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB112_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s34 ; GCN3-NEXT: v_mov_b32_e32 v5, s35 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s7 ; GCN3-NEXT: v_mov_b32_e32 v7, s6 ; GCN3-NEXT: .LBB112_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB112_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB112_2 ; GCN3-NEXT: .LBB112_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_umin_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB113_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v2, s4 ; GCN1-NEXT: v_mov_b32_e32 v3, s5 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: .LBB113_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB113_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB113_6 ; GCN1-NEXT: .LBB113_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB113_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB113_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umin_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB113_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v2, s4 ; GCN2-NEXT: v_mov_b32_e32 v3, s5 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: .LBB113_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB113_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB113_6 ; GCN2-NEXT: .LBB113_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB113_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB113_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umin_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB113_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s4 ; GCN3-NEXT: v_mov_b32_e32 v3, s5 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s7 ; GCN3-NEXT: v_mov_b32_e32 v5, s6 ; GCN3-NEXT: .LBB113_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB113_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB113_6 ; GCN3-NEXT: .LBB113_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB113_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB113_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw umin ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_umin_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB114_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: v_mov_b32_e32 v3, s35 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: .LBB114_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB114_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB114_6 ; GCN1-NEXT: .LBB114_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB114_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB114_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umin_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB114_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: v_mov_b32_e32 v3, s35 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: .LBB114_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB114_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB114_6 ; GCN2-NEXT: .LBB114_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB114_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB114_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umin_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB114_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: v_mov_b32_e32 v3, s35 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s7 ; GCN3-NEXT: v_mov_b32_e32 v5, s6 ; GCN3-NEXT: .LBB114_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB114_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB114_6 ; GCN3-NEXT: .LBB114_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB114_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB114_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw umin ptr %gep, i64 %in seq_cst ret i64 %result } define void @flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB115_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB115_4 ; GCN1-NEXT: .LBB115_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB115_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB115_2 ; GCN1-NEXT: .LBB115_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB115_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB115_4 ; GCN2-NEXT: .LBB115_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB115_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB115_2 ; GCN2-NEXT: .LBB115_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB115_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB115_4 ; GCN3-NEXT: .LBB115_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB115_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB115_2 ; GCN3-NEXT: .LBB115_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB116_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB116_4 ; GCN1-NEXT: .LBB116_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB116_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB116_2 ; GCN1-NEXT: .LBB116_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB116_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB116_4 ; GCN2-NEXT: .LBB116_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB116_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB116_2 ; GCN2-NEXT: .LBB116_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB116_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB116_4 ; GCN3-NEXT: .LBB116_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB116_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB116_2 ; GCN3-NEXT: .LBB116_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw umin ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } ; --------------------------------------------------------------------- ; atomicrmw min ; --------------------------------------------------------------------- define void @flat_atomic_min_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_min_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB117_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB117_6 ; GCN1-NEXT: .LBB117_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB117_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB117_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB117_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB117_2 ; GCN1-NEXT: .LBB117_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_min_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB117_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB117_6 ; GCN2-NEXT: .LBB117_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB117_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB117_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB117_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB117_2 ; GCN2-NEXT: .LBB117_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_min_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB117_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB117_6 ; GCN3-NEXT: .LBB117_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB117_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB117_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB117_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB117_2 ; GCN3-NEXT: .LBB117_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_min_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_min_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB118_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB118_6 ; GCN1-NEXT: .LBB118_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB118_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB118_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB118_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB118_2 ; GCN1-NEXT: .LBB118_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_min_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB118_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB118_6 ; GCN2-NEXT: .LBB118_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB118_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB118_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB118_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB118_2 ; GCN2-NEXT: .LBB118_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_min_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB118_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB118_6 ; GCN3-NEXT: .LBB118_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB118_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB118_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB118_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB118_2 ; GCN3-NEXT: .LBB118_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_min_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_min_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB119_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v5, v[4:5] ; GCN1-NEXT: flat_load_dword v4, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB119_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB119_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: .LBB119_4: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB119_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[4:5], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: .LBB119_6: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v0, v4 ; GCN1-NEXT: v_mov_b32_e32 v1, v5 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_min_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB119_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v5, v[4:5] ; GCN2-NEXT: flat_load_dword v4, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB119_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB119_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: .LBB119_4: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB119_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[4:5], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: .LBB119_6: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v0, v4 ; GCN2-NEXT: v_mov_b32_e32 v1, v5 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_min_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB119_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB119_6 ; GCN3-NEXT: .LBB119_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB119_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB119_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB119_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB119_2 ; GCN3-NEXT: .LBB119_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw min ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_min_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_min_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB120_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB120_6 ; GCN1-NEXT: .LBB120_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB120_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[4:5] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB120_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB120_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB120_2 ; GCN1-NEXT: .LBB120_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_min_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB120_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB120_6 ; GCN2-NEXT: .LBB120_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB120_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[4:5] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB120_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB120_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB120_2 ; GCN2-NEXT: .LBB120_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_min_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB120_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB120_6 ; GCN3-NEXT: .LBB120_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB120_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB120_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB120_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB120_2 ; GCN3-NEXT: .LBB120_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw min ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_min_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB121_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccnz .LBB121_6 ; GCN1-NEXT: .LBB121_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB121_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v4, s4 ; GCN1-NEXT: v_mov_b32_e32 v5, s5 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s7 ; GCN1-NEXT: v_mov_b32_e32 v7, s6 ; GCN1-NEXT: .LBB121_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB121_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB121_2 ; GCN1-NEXT: .LBB121_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_min_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB121_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccnz .LBB121_6 ; GCN2-NEXT: .LBB121_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB121_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v4, s4 ; GCN2-NEXT: v_mov_b32_e32 v5, s5 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s7 ; GCN2-NEXT: v_mov_b32_e32 v7, s6 ; GCN2-NEXT: .LBB121_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB121_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB121_2 ; GCN2-NEXT: .LBB121_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_min_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB121_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccnz .LBB121_6 ; GCN3-NEXT: .LBB121_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB121_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s4 ; GCN3-NEXT: v_mov_b32_e32 v5, s5 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s7 ; GCN3-NEXT: v_mov_b32_e32 v7, s6 ; GCN3-NEXT: .LBB121_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB121_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB121_2 ; GCN3-NEXT: .LBB121_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_min_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB122_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccnz .LBB122_6 ; GCN1-NEXT: .LBB122_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB122_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v4, s34 ; GCN1-NEXT: v_mov_b32_e32 v5, s35 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s7 ; GCN1-NEXT: v_mov_b32_e32 v7, s6 ; GCN1-NEXT: .LBB122_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB122_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB122_2 ; GCN1-NEXT: .LBB122_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_min_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB122_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccnz .LBB122_6 ; GCN2-NEXT: .LBB122_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB122_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v4, s34 ; GCN2-NEXT: v_mov_b32_e32 v5, s35 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s7 ; GCN2-NEXT: v_mov_b32_e32 v7, s6 ; GCN2-NEXT: .LBB122_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB122_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB122_2 ; GCN2-NEXT: .LBB122_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_min_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB122_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccnz .LBB122_6 ; GCN3-NEXT: .LBB122_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB122_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s34 ; GCN3-NEXT: v_mov_b32_e32 v5, s35 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s7 ; GCN3-NEXT: v_mov_b32_e32 v7, s6 ; GCN3-NEXT: .LBB122_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB122_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB122_2 ; GCN3-NEXT: .LBB122_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_min_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB123_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v2, s4 ; GCN1-NEXT: v_mov_b32_e32 v3, s5 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: .LBB123_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB123_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB123_6 ; GCN1-NEXT: .LBB123_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB123_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB123_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_min_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB123_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v2, s4 ; GCN2-NEXT: v_mov_b32_e32 v3, s5 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: .LBB123_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB123_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB123_6 ; GCN2-NEXT: .LBB123_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB123_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB123_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_min_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB123_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s4 ; GCN3-NEXT: v_mov_b32_e32 v3, s5 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s7 ; GCN3-NEXT: v_mov_b32_e32 v5, s6 ; GCN3-NEXT: .LBB123_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB123_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB123_6 ; GCN3-NEXT: .LBB123_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB123_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB123_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw min ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_min_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB124_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: v_mov_b32_e32 v3, s35 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: .LBB124_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB124_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB124_6 ; GCN1-NEXT: .LBB124_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB124_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB124_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_min_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB124_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: v_mov_b32_e32 v3, s35 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: .LBB124_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB124_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB124_6 ; GCN2-NEXT: .LBB124_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB124_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB124_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_min_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB124_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: v_mov_b32_e32 v3, s35 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s7 ; GCN3-NEXT: v_mov_b32_e32 v5, s6 ; GCN3-NEXT: .LBB124_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB124_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB124_6 ; GCN3-NEXT: .LBB124_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB124_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB124_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw min ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %index) { ; GCN1-LABEL: atomic_min_i64_addr64_offset: ; GCN1: ; %bb.0: ; %entry ; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GCN1-NEXT: s_mov_b32 s14, -1 ; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd ; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f ; GCN1-NEXT: s_mov_b32 s15, 0xe8f000 ; GCN1-NEXT: s_add_u32 s12, s12, s11 ; GCN1-NEXT: s_addc_u32 s13, s13, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 ; GCN1-NEXT: s_add_u32 s0, s0, s4 ; GCN1-NEXT: s_addc_u32 s1, s1, s5 ; GCN1-NEXT: s_add_u32 s0, s0, 32 ; GCN1-NEXT: s_addc_u32 s1, s1, 0 ; GCN1-NEXT: s_cmp_eq_u32 s1, s8 ; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN1-NEXT: s_mov_b64 s[4:5], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB125_3 ; GCN1-NEXT: ; %bb.1: ; %Flow6 ; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN1-NEXT: s_cbranch_vccnz .LBB125_6 ; GCN1-NEXT: .LBB125_2: ; %atomicrmw.phi ; GCN1-NEXT: s_endpgm ; GCN1-NEXT: .LBB125_3: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v5, s1 ; GCN1-NEXT: v_mov_b32_e32 v4, s0 ; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN1-NEXT: s_mov_b64 s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s3 ; GCN1-NEXT: v_mov_b32_e32 v7, s2 ; GCN1-NEXT: .LBB125_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB125_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_branch .LBB125_2 ; GCN1-NEXT: .LBB125_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s2 ; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: s_add_i32 s0, s0, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s0 ; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s3 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen ; GCN1-NEXT: s_endpgm ; ; GCN2-LABEL: atomic_min_i64_addr64_offset: ; GCN2: ; %bb.0: ; %entry ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 ; GCN2-NEXT: s_mov_b32 s90, -1 ; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 ; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 ; GCN2-NEXT: s_add_u32 s88, s88, s11 ; GCN2-NEXT: s_addc_u32 s89, s89, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 ; GCN2-NEXT: s_add_u32 s0, s0, s4 ; GCN2-NEXT: s_addc_u32 s1, s1, s5 ; GCN2-NEXT: s_add_u32 s0, s0, 32 ; GCN2-NEXT: s_addc_u32 s1, s1, 0 ; GCN2-NEXT: s_cmp_eq_u32 s1, s8 ; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN2-NEXT: s_mov_b64 s[4:5], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB125_3 ; GCN2-NEXT: ; %bb.1: ; %Flow6 ; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN2-NEXT: s_cbranch_vccnz .LBB125_6 ; GCN2-NEXT: .LBB125_2: ; %atomicrmw.phi ; GCN2-NEXT: s_endpgm ; GCN2-NEXT: .LBB125_3: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v5, s1 ; GCN2-NEXT: v_mov_b32_e32 v4, s0 ; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN2-NEXT: s_mov_b64 s[4:5], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s3 ; GCN2-NEXT: v_mov_b32_e32 v7, s2 ; GCN2-NEXT: .LBB125_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB125_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_branch .LBB125_2 ; GCN2-NEXT: .LBB125_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: s_add_i32 s0, s0, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s0 ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s2 ; GCN2-NEXT: v_mov_b32_e32 v4, s3 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: s_endpgm ; ; GCN3-LABEL: atomic_min_i64_addr64_offset: ; GCN3: ; %bb.0: ; %entry ; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GCN3-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34 ; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GCN3-NEXT: s_mov_b32 s14, -1 ; GCN3-NEXT: s_mov_b32 s15, 0xe00000 ; GCN3-NEXT: s_add_u32 s12, s12, s11 ; GCN3-NEXT: s_addc_u32 s13, s13, 0 ; GCN3-NEXT: s_waitcnt lgkmcnt(0) ; GCN3-NEXT: s_lshl_b64 s[4:5], s[8:9], 3 ; GCN3-NEXT: s_add_u32 s0, s0, s4 ; GCN3-NEXT: s_addc_u32 s1, s1, s5 ; GCN3-NEXT: s_add_u32 s0, s0, 32 ; GCN3-NEXT: s_mov_b64 s[6:7], src_private_base ; GCN3-NEXT: s_addc_u32 s1, s1, 0 ; GCN3-NEXT: s_cmp_eq_u32 s1, s7 ; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN3-NEXT: s_mov_b64 s[4:5], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB125_3 ; GCN3-NEXT: ; %bb.1: ; %Flow6 ; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN3-NEXT: s_cbranch_vccnz .LBB125_6 ; GCN3-NEXT: .LBB125_2: ; %atomicrmw.phi ; GCN3-NEXT: s_endpgm ; GCN3-NEXT: .LBB125_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v5, s1 ; GCN3-NEXT: v_mov_b32_e32 v4, s0 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[4:5], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s3 ; GCN3-NEXT: v_mov_b32_e32 v7, s2 ; GCN3-NEXT: .LBB125_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB125_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_branch .LBB125_2 ; GCN3-NEXT: .LBB125_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s2 ; GCN3-NEXT: v_mov_b32_e32 v3, s3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4 ; GCN3-NEXT: s_endpgm entry: %ptr = getelementptr i64, ptr %out, i64 %index %gep = getelementptr i64, ptr %ptr, i64 4 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst ret void } define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) { ; GCN1-LABEL: atomic_min_i64_ret_addr64_offset: ; GCN1: ; %bb.0: ; %entry ; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GCN1-NEXT: s_mov_b32 s18, -1 ; GCN1-NEXT: s_mov_b32 s19, 0xe8f000 ; GCN1-NEXT: s_add_u32 s16, s16, s11 ; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9 ; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41 ; GCN1-NEXT: s_addc_u32 s17, s17, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN1-NEXT: s_add_u32 s0, s8, s0 ; GCN1-NEXT: s_addc_u32 s1, s9, s1 ; GCN1-NEXT: s_add_u32 s0, s0, 32 ; GCN1-NEXT: s_addc_u32 s1, s1, 0 ; GCN1-NEXT: s_cmp_eq_u32 s1, s2 ; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN1-NEXT: s_cbranch_vccz .LBB126_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v3, s1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN1-NEXT: s_mov_b64 s[2:3], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s13 ; GCN1-NEXT: v_mov_b32_e32 v5, s12 ; GCN1-NEXT: .LBB126_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN1-NEXT: s_cbranch_execnz .LBB126_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN1-NEXT: s_branch .LBB126_6 ; GCN1-NEXT: .LBB126_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB126_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s12 ; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: s_add_i32 s0, s0, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s0 ; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s13 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen ; GCN1-NEXT: .LBB126_6: ; %atomicrmw.phi ; GCN1-NEXT: v_mov_b32_e32 v2, s10 ; GCN1-NEXT: v_mov_b32_e32 v3, s11 ; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN1-NEXT: s_endpgm ; ; GCN2-LABEL: atomic_min_i64_ret_addr64_offset: ; GCN2: ; %bb.0: ; %entry ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 ; GCN2-NEXT: s_mov_b32 s90, -1 ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 ; GCN2-NEXT: s_add_u32 s88, s88, s11 ; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104 ; GCN2-NEXT: s_addc_u32 s89, s89, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN2-NEXT: s_add_u32 s0, s8, s0 ; GCN2-NEXT: s_addc_u32 s1, s9, s1 ; GCN2-NEXT: s_add_u32 s0, s0, 32 ; GCN2-NEXT: s_addc_u32 s1, s1, 0 ; GCN2-NEXT: s_cmp_eq_u32 s1, s2 ; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN2-NEXT: s_cbranch_vccz .LBB126_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v3, s1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN2-NEXT: s_mov_b64 s[2:3], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s13 ; GCN2-NEXT: v_mov_b32_e32 v5, s12 ; GCN2-NEXT: .LBB126_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN2-NEXT: s_cbranch_execnz .LBB126_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN2-NEXT: s_branch .LBB126_6 ; GCN2-NEXT: .LBB126_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB126_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: s_add_i32 s0, s0, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s0 ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s12 ; GCN2-NEXT: v_mov_b32_e32 v4, s13 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen ; GCN2-NEXT: .LBB126_6: ; %atomicrmw.phi ; GCN2-NEXT: v_mov_b32_e32 v2, s10 ; GCN2-NEXT: v_mov_b32_e32 v3, s11 ; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN2-NEXT: s_endpgm ; ; GCN3-LABEL: atomic_min_i64_ret_addr64_offset: ; GCN3: ; %bb.0: ; %entry ; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GCN3-NEXT: s_mov_b32 s18, -1 ; GCN3-NEXT: s_mov_b32 s19, 0xe00000 ; GCN3-NEXT: s_add_u32 s16, s16, s11 ; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GCN3-NEXT: s_addc_u32 s17, s17, 0 ; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base ; GCN3-NEXT: s_waitcnt lgkmcnt(0) ; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN3-NEXT: s_add_u32 s0, s8, s0 ; GCN3-NEXT: s_addc_u32 s1, s9, s1 ; GCN3-NEXT: s_add_u32 s0, s0, 32 ; GCN3-NEXT: s_addc_u32 s1, s1, 0 ; GCN3-NEXT: s_cmp_eq_u32 s1, s3 ; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN3-NEXT: s_cbranch_vccz .LBB126_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v3, s1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[2:3], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s13 ; GCN3-NEXT: v_mov_b32_e32 v5, s12 ; GCN3-NEXT: .LBB126_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN3-NEXT: s_cbranch_execnz .LBB126_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN3-NEXT: s_branch .LBB126_6 ; GCN3-NEXT: .LBB126_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB126_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s12 ; GCN3-NEXT: v_mov_b32_e32 v3, s13 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4 ; GCN3-NEXT: .LBB126_6: ; %atomicrmw.phi ; GCN3-NEXT: v_mov_b32_e32 v2, s10 ; GCN3-NEXT: v_mov_b32_e32 v3, s11 ; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN3-NEXT: s_endpgm entry: %ptr = getelementptr i64, ptr %out, i64 %index %gep = getelementptr i64, ptr %ptr, i64 4 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst store i64 %tmp0, ptr %out2 ret void } define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) { ; GCN1-LABEL: atomic_min_i64: ; GCN1: ; %bb.0: ; %entry ; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d ; GCN1-NEXT: s_mov_b32 s14, -1 ; GCN1-NEXT: s_mov_b32 s15, 0xe8f000 ; GCN1-NEXT: s_add_u32 s12, s12, s11 ; GCN1-NEXT: s_addc_u32 s13, s13, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s1, s4 ; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN1-NEXT: s_mov_b64 s[4:5], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB127_3 ; GCN1-NEXT: ; %bb.1: ; %Flow5 ; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN1-NEXT: s_cbranch_vccnz .LBB127_6 ; GCN1-NEXT: .LBB127_2: ; %atomicrmw.phi ; GCN1-NEXT: s_endpgm ; GCN1-NEXT: .LBB127_3: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v5, s1 ; GCN1-NEXT: v_mov_b32_e32 v4, s0 ; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN1-NEXT: s_mov_b64 s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s3 ; GCN1-NEXT: v_mov_b32_e32 v7, s2 ; GCN1-NEXT: .LBB127_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB127_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_branch .LBB127_2 ; GCN1-NEXT: .LBB127_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s2 ; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: s_add_i32 s0, s0, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s0 ; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s3 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen ; GCN1-NEXT: s_endpgm ; ; GCN2-LABEL: atomic_min_i64: ; GCN2: ; %bb.0: ; %entry ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 ; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf4 ; GCN2-NEXT: s_mov_b32 s90, -1 ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 ; GCN2-NEXT: s_add_u32 s88, s88, s11 ; GCN2-NEXT: s_addc_u32 s89, s89, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s1, s4 ; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN2-NEXT: s_mov_b64 s[4:5], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB127_3 ; GCN2-NEXT: ; %bb.1: ; %Flow5 ; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN2-NEXT: s_cbranch_vccnz .LBB127_6 ; GCN2-NEXT: .LBB127_2: ; %atomicrmw.phi ; GCN2-NEXT: s_endpgm ; GCN2-NEXT: .LBB127_3: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v5, s1 ; GCN2-NEXT: v_mov_b32_e32 v4, s0 ; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN2-NEXT: s_mov_b64 s[4:5], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s3 ; GCN2-NEXT: v_mov_b32_e32 v7, s2 ; GCN2-NEXT: .LBB127_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB127_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_branch .LBB127_2 ; GCN2-NEXT: .LBB127_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: s_add_i32 s0, s0, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s0 ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s2 ; GCN2-NEXT: v_mov_b32_e32 v4, s3 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: s_endpgm ; ; GCN3-LABEL: atomic_min_i64: ; GCN3: ; %bb.0: ; %entry ; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GCN3-NEXT: s_mov_b32 s14, -1 ; GCN3-NEXT: s_mov_b32 s15, 0xe00000 ; GCN3-NEXT: s_add_u32 s12, s12, s11 ; GCN3-NEXT: s_addc_u32 s13, s13, 0 ; GCN3-NEXT: s_mov_b64 s[6:7], src_private_base ; GCN3-NEXT: s_waitcnt lgkmcnt(0) ; GCN3-NEXT: s_cmp_eq_u32 s1, s7 ; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GCN3-NEXT: s_mov_b64 s[4:5], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB127_3 ; GCN3-NEXT: ; %bb.1: ; %Flow5 ; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5] ; GCN3-NEXT: s_cbranch_vccnz .LBB127_6 ; GCN3-NEXT: .LBB127_2: ; %atomicrmw.phi ; GCN3-NEXT: s_endpgm ; GCN3-NEXT: .LBB127_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v5, s1 ; GCN3-NEXT: v_mov_b32_e32 v4, s0 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[4:5], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s3 ; GCN3-NEXT: v_mov_b32_e32 v7, s2 ; GCN3-NEXT: .LBB127_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB127_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_branch .LBB127_2 ; GCN3-NEXT: .LBB127_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s2 ; GCN3-NEXT: v_mov_b32_e32 v3, s3 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4 ; GCN3-NEXT: s_endpgm entry: %tmp0 = atomicrmw min ptr %out, i64 %in seq_cst ret void } define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) { ; GCN1-LABEL: atomic_min_i64_ret_addr64: ; GCN1: ; %bb.0: ; %entry ; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GCN1-NEXT: s_mov_b32 s18, -1 ; GCN1-NEXT: s_mov_b32 s19, 0xe8f000 ; GCN1-NEXT: s_add_u32 s16, s16, s11 ; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9 ; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41 ; GCN1-NEXT: s_addc_u32 s17, s17, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN1-NEXT: s_add_u32 s0, s8, s0 ; GCN1-NEXT: s_addc_u32 s1, s9, s1 ; GCN1-NEXT: s_cmp_eq_u32 s1, s2 ; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN1-NEXT: s_cbranch_vccz .LBB128_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_mov_b32_e32 v3, s1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN1-NEXT: s_mov_b64 s[2:3], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s13 ; GCN1-NEXT: v_mov_b32_e32 v5, s12 ; GCN1-NEXT: .LBB128_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN1-NEXT: s_cbranch_execnz .LBB128_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN1-NEXT: s_branch .LBB128_6 ; GCN1-NEXT: .LBB128_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB128_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s12 ; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s0 ; GCN1-NEXT: s_add_i32 s0, s0, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s0 ; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s13 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen ; GCN1-NEXT: .LBB128_6: ; %atomicrmw.phi ; GCN1-NEXT: v_mov_b32_e32 v2, s10 ; GCN1-NEXT: v_mov_b32_e32 v3, s11 ; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN1-NEXT: s_endpgm ; ; GCN2-LABEL: atomic_min_i64_ret_addr64: ; GCN2: ; %bb.0: ; %entry ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 ; GCN2-NEXT: s_mov_b32 s90, -1 ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 ; GCN2-NEXT: s_add_u32 s88, s88, s11 ; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104 ; GCN2-NEXT: s_addc_u32 s89, s89, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN2-NEXT: s_add_u32 s0, s8, s0 ; GCN2-NEXT: s_addc_u32 s1, s9, s1 ; GCN2-NEXT: s_cmp_eq_u32 s1, s2 ; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN2-NEXT: s_cbranch_vccz .LBB128_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_mov_b32_e32 v3, s1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN2-NEXT: s_mov_b64 s[2:3], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s13 ; GCN2-NEXT: v_mov_b32_e32 v5, s12 ; GCN2-NEXT: .LBB128_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN2-NEXT: s_cbranch_execnz .LBB128_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN2-NEXT: s_branch .LBB128_6 ; GCN2-NEXT: .LBB128_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB128_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s0 ; GCN2-NEXT: s_add_i32 s0, s0, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s0 ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s12 ; GCN2-NEXT: v_mov_b32_e32 v4, s13 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen ; GCN2-NEXT: .LBB128_6: ; %atomicrmw.phi ; GCN2-NEXT: v_mov_b32_e32 v2, s10 ; GCN2-NEXT: v_mov_b32_e32 v3, s11 ; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN2-NEXT: s_endpgm ; ; GCN3-LABEL: atomic_min_i64_ret_addr64: ; GCN3: ; %bb.0: ; %entry ; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GCN3-NEXT: s_mov_b32 s18, -1 ; GCN3-NEXT: s_mov_b32 s19, 0xe00000 ; GCN3-NEXT: s_add_u32 s16, s16, s11 ; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GCN3-NEXT: s_addc_u32 s17, s17, 0 ; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base ; GCN3-NEXT: s_waitcnt lgkmcnt(0) ; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 ; GCN3-NEXT: s_add_u32 s0, s8, s0 ; GCN3-NEXT: s_addc_u32 s1, s9, s1 ; GCN3-NEXT: s_cmp_eq_u32 s1, s3 ; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3] ; GCN3-NEXT: s_cbranch_vccz .LBB128_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v3, s1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[2:3], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s13 ; GCN3-NEXT: v_mov_b32_e32 v5, s12 ; GCN3-NEXT: .LBB128_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GCN3-NEXT: s_cbranch_execnz .LBB128_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[2:3] ; GCN3-NEXT: s_branch .LBB128_6 ; GCN3-NEXT: .LBB128_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB128_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s0 ; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s12 ; GCN3-NEXT: v_mov_b32_e32 v3, s13 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4 ; GCN3-NEXT: .LBB128_6: ; %atomicrmw.phi ; GCN3-NEXT: v_mov_b32_e32 v2, s10 ; GCN3-NEXT: v_mov_b32_e32 v3, s11 ; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GCN3-NEXT: s_endpgm entry: %ptr = getelementptr i64, ptr %out, i64 %index %tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst store i64 %tmp0, ptr %out2 ret void } define void @flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB129_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB129_4 ; GCN1-NEXT: .LBB129_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB129_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB129_2 ; GCN1-NEXT: .LBB129_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB129_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB129_4 ; GCN2-NEXT: .LBB129_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB129_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB129_2 ; GCN2-NEXT: .LBB129_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB129_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB129_4 ; GCN3-NEXT: .LBB129_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB129_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB129_2 ; GCN3-NEXT: .LBB129_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB130_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB130_4 ; GCN1-NEXT: .LBB130_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB130_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB130_2 ; GCN1-NEXT: .LBB130_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB130_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB130_4 ; GCN2-NEXT: .LBB130_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB130_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB130_2 ; GCN2-NEXT: .LBB130_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB130_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB130_4 ; GCN3-NEXT: .LBB130_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB130_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB130_2 ; GCN3-NEXT: .LBB130_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw min ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } ; --------------------------------------------------------------------- ; atomicrmw uinc_wrap ; --------------------------------------------------------------------- define void @flat_atomic_uinc_wrap_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB131_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB131_6 ; GCN1-NEXT: .LBB131_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB131_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB131_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v6 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB131_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB131_2 ; GCN1-NEXT: .LBB131_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc ; GCN1-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB131_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB131_6 ; GCN2-NEXT: .LBB131_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB131_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB131_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v6 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB131_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB131_2 ; GCN2-NEXT: .LBB131_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc ; GCN2-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB131_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB131_6 ; GCN3-NEXT: .LBB131_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB131_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB131_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 1, v6 ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB131_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB131_2 ; GCN3-NEXT: .LBB131_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_uinc_wrap_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB132_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB132_6 ; GCN1-NEXT: .LBB132_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB132_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB132_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v6 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB132_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB132_2 ; GCN1-NEXT: .LBB132_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc ; GCN1-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB132_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB132_6 ; GCN2-NEXT: .LBB132_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB132_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB132_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v6 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB132_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB132_2 ; GCN2-NEXT: .LBB132_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc ; GCN2-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB132_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB132_6 ; GCN3-NEXT: .LBB132_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB132_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB132_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 1, v6 ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB132_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB132_2 ; GCN3-NEXT: .LBB132_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_uinc_wrap_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB133_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v5, v[4:5] ; GCN1-NEXT: flat_load_dword v4, v[0:1] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB133_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v6 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB133_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: .LBB133_4: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB133_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v4 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v5, vcc ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen ; GCN1-NEXT: .LBB133_6: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: v_mov_b32_e32 v0, v4 ; GCN1-NEXT: v_mov_b32_e32 v1, v5 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB133_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v5, v[4:5] ; GCN2-NEXT: flat_load_dword v4, v[0:1] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB133_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v6 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB133_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: .LBB133_4: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB133_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v4 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v5, vcc ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen ; GCN2-NEXT: .LBB133_6: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: v_mov_b32_e32 v0, v4 ; GCN2-NEXT: v_mov_b32_e32 v1, v5 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB133_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB133_6 ; GCN3-NEXT: .LBB133_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB133_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB133_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v8 ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB133_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB133_2 ; GCN3-NEXT: .LBB133_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_uinc_wrap_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB134_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB134_6 ; GCN1-NEXT: .LBB134_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB134_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[4:5] ; GCN1-NEXT: s_mov_b64 s[6:7], 0 ; GCN1-NEXT: .LBB134_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v8 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v9, vcc ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN1-NEXT: s_cbranch_execnz .LBB134_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB134_2 ; GCN1-NEXT: .LBB134_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc ; GCN1-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB134_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB134_6 ; GCN2-NEXT: .LBB134_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB134_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[4:5] ; GCN2-NEXT: s_mov_b64 s[6:7], 0 ; GCN2-NEXT: .LBB134_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v8 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v9, vcc ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN2-NEXT: s_cbranch_execnz .LBB134_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB134_2 ; GCN2-NEXT: .LBB134_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc ; GCN2-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB134_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB134_6 ; GCN3-NEXT: .LBB134_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB134_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[6:7], 0 ; GCN3-NEXT: .LBB134_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v8 ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] ; GCN3-NEXT: s_cbranch_execnz .LBB134_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB134_2 ; GCN3-NEXT: .LBB134_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB135_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccnz .LBB135_6 ; GCN1-NEXT: .LBB135_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB135_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v4, s4 ; GCN1-NEXT: v_mov_b32_e32 v5, s5 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: .LBB135_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v2 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB135_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB135_2 ; GCN1-NEXT: .LBB135_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc ; GCN1-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v0, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB135_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccnz .LBB135_6 ; GCN2-NEXT: .LBB135_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB135_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v4, s4 ; GCN2-NEXT: v_mov_b32_e32 v5, s5 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: .LBB135_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v2 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB135_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB135_2 ; GCN2-NEXT: .LBB135_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc ; GCN2-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v0, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB135_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccnz .LBB135_6 ; GCN3-NEXT: .LBB135_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB135_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s4 ; GCN3-NEXT: v_mov_b32_e32 v5, s5 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: .LBB135_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v2 ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB135_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB135_2 ; GCN3-NEXT: .LBB135_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v3, vcc, 1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v4, vcc ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_mov_b64 s[36:37], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB136_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccnz .LBB136_6 ; GCN1-NEXT: .LBB136_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB136_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v4, s34 ; GCN1-NEXT: v_mov_b32_e32 v5, s35 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: .LBB136_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v2 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB136_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB136_2 ; GCN1-NEXT: .LBB136_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc ; GCN1-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v0, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_mov_b64 s[36:37], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB136_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccnz .LBB136_6 ; GCN2-NEXT: .LBB136_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB136_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v4, s34 ; GCN2-NEXT: v_mov_b32_e32 v5, s35 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: .LBB136_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v2 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB136_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB136_2 ; GCN2-NEXT: .LBB136_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc ; GCN2-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v0, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_mov_b64 s[36:37], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB136_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccnz .LBB136_6 ; GCN3-NEXT: .LBB136_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB136_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s34 ; GCN3-NEXT: v_mov_b32_e32 v5, s35 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: .LBB136_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v2 ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB136_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB136_2 ; GCN3-NEXT: .LBB136_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v3, vcc, 1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v4, vcc ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB137_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v2, s4 ; GCN1-NEXT: v_mov_b32_e32 v3, s5 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[34:35], 0 ; GCN1-NEXT: .LBB137_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v1 ; GCN1-NEXT: v_mov_b32_e32 v6, v0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v6 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7] ; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_cbranch_execnz .LBB137_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN1-NEXT: s_branch .LBB137_6 ; GCN1-NEXT: .LBB137_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB137_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB137_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB137_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v2, s4 ; GCN2-NEXT: v_mov_b32_e32 v3, s5 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[34:35], 0 ; GCN2-NEXT: .LBB137_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v1 ; GCN2-NEXT: v_mov_b32_e32 v6, v0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v6 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7] ; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_cbranch_execnz .LBB137_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN2-NEXT: s_branch .LBB137_6 ; GCN2-NEXT: .LBB137_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB137_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB137_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB137_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s4 ; GCN3-NEXT: v_mov_b32_e32 v3, s5 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[34:35], 0 ; GCN3-NEXT: .LBB137_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v7, v1 ; GCN3-NEXT: v_mov_b32_e32 v6, v0 ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v6 ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7] ; GCN3-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_cbranch_execnz .LBB137_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] ; GCN3-NEXT: s_branch .LBB137_6 ; GCN3-NEXT: .LBB137_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB137_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v3, vcc, 1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB137_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s34, s4, 32 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN1-NEXT: s_cbranch_vccz .LBB138_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s36, s34, 4 ; GCN1-NEXT: s_addc_u32 s37, s35, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s36 ; GCN1-NEXT: v_mov_b32_e32 v1, s37 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: v_mov_b32_e32 v3, s35 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[36:37], 0 ; GCN1-NEXT: .LBB138_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v1 ; GCN1-NEXT: v_mov_b32_e32 v6, v0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v6 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7] ; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_cbranch_execnz .LBB138_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN1-NEXT: s_branch .LBB138_6 ; GCN1-NEXT: .LBB138_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB138_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB138_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s34, s4, 32 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN2-NEXT: s_cbranch_vccz .LBB138_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s36, s34, 4 ; GCN2-NEXT: s_addc_u32 s37, s35, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s36 ; GCN2-NEXT: v_mov_b32_e32 v1, s37 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: v_mov_b32_e32 v3, s35 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[36:37], 0 ; GCN2-NEXT: .LBB138_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v1 ; GCN2-NEXT: v_mov_b32_e32 v6, v0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v6 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7] ; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_cbranch_execnz .LBB138_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN2-NEXT: s_branch .LBB138_6 ; GCN2-NEXT: .LBB138_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB138_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB138_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s34, s4, 32 ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base ; GCN3-NEXT: s_addc_u32 s35, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] ; GCN3-NEXT: s_cbranch_vccz .LBB138_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: v_mov_b32_e32 v3, s35 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[36:37], 0 ; GCN3-NEXT: .LBB138_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v7, v1 ; GCN3-NEXT: v_mov_b32_e32 v6, v0 ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v6 ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7] ; GCN3-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_cbranch_execnz .LBB138_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] ; GCN3-NEXT: s_branch .LBB138_6 ; GCN3-NEXT: .LBB138_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB138_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v3, vcc, 1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB138_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst ret i64 %result } define void @flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB139_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB139_4 ; GCN1-NEXT: .LBB139_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB139_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB139_2 ; GCN1-NEXT: .LBB139_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc ; GCN1-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB139_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB139_4 ; GCN2-NEXT: .LBB139_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB139_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB139_2 ; GCN2-NEXT: .LBB139_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc ; GCN2-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB139_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB139_4 ; GCN3-NEXT: .LBB139_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB139_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB139_2 ; GCN3-NEXT: .LBB139_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB140_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB140_4 ; GCN1-NEXT: .LBB140_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB140_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB140_2 ; GCN1-NEXT: .LBB140_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc ; GCN1-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB140_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB140_4 ; GCN2-NEXT: .LBB140_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB140_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB140_2 ; GCN2-NEXT: .LBB140_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc ; GCN2-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB140_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB140_4 ; GCN3-NEXT: .LBB140_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB140_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB140_2 ; GCN3-NEXT: .LBB140_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } ; --------------------------------------------------------------------- ; atomicrmw udec_wrap ; --------------------------------------------------------------------- define void @flat_atomic_udec_wrap_i64_noret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB141_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN1-NEXT: s_cbranch_execnz .LBB141_6 ; GCN1-NEXT: .LBB141_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB141_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[10:11], 0 ; GCN1-NEXT: .LBB141_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3] ; GCN1-NEXT: v_add_i32_e64 v4, s[6:7], -1, v6 ; GCN1-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[10:11] ; GCN1-NEXT: s_cbranch_execnz .LBB141_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[10:11] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN1-NEXT: s_cbranch_execz .LBB141_2 ; GCN1-NEXT: .LBB141_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN1-NEXT: v_add_i32_e64 v0, s[6:7], -1, v0 ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN1-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB141_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN2-NEXT: s_cbranch_execnz .LBB141_6 ; GCN2-NEXT: .LBB141_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB141_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[10:11], 0 ; GCN2-NEXT: .LBB141_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3] ; GCN2-NEXT: v_add_u32_e64 v4, s[6:7], -1, v6 ; GCN2-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[10:11] ; GCN2-NEXT: s_cbranch_execnz .LBB141_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[10:11] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN2-NEXT: s_cbranch_execz .LBB141_2 ; GCN2-NEXT: .LBB141_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN2-NEXT: v_add_u32_e64 v0, s[6:7], -1, v0 ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN2-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB141_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN3-NEXT: s_cbranch_execnz .LBB141_6 ; GCN3-NEXT: .LBB141_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB141_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[10:11], 0 ; GCN3-NEXT: .LBB141_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3] ; GCN3-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v6 ; GCN3-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[10:11] ; GCN3-NEXT: s_cbranch_execnz .LBB141_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[10:11] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN3-NEXT: s_cbranch_execz .LBB141_2 ; GCN3-NEXT: .LBB141_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v0 ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v1, s[6:7] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst ret void } define void @flat_atomic_udec_wrap_i64_noret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB142_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN1-NEXT: s_cbranch_execnz .LBB142_6 ; GCN1-NEXT: .LBB142_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB142_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v7, v[4:5] ; GCN1-NEXT: flat_load_dword v6, v[0:1] ; GCN1-NEXT: s_mov_b64 s[10:11], 0 ; GCN1-NEXT: .LBB142_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3] ; GCN1-NEXT: v_add_i32_e64 v4, s[6:7], -1, v6 ; GCN1-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[10:11] ; GCN1-NEXT: s_cbranch_execnz .LBB142_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[10:11] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN1-NEXT: s_cbranch_execz .LBB142_2 ; GCN1-NEXT: .LBB142_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN1-NEXT: v_add_i32_e64 v0, s[6:7], -1, v0 ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN1-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB142_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN2-NEXT: s_cbranch_execnz .LBB142_6 ; GCN2-NEXT: .LBB142_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB142_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v7, v[4:5] ; GCN2-NEXT: flat_load_dword v6, v[0:1] ; GCN2-NEXT: s_mov_b64 s[10:11], 0 ; GCN2-NEXT: .LBB142_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3] ; GCN2-NEXT: v_add_u32_e64 v4, s[6:7], -1, v6 ; GCN2-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[10:11] ; GCN2-NEXT: s_cbranch_execnz .LBB142_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[10:11] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN2-NEXT: s_cbranch_execz .LBB142_2 ; GCN2-NEXT: .LBB142_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN2-NEXT: v_add_u32_e64 v0, s[6:7], -1, v0 ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN2-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB142_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN3-NEXT: s_cbranch_execnz .LBB142_6 ; GCN3-NEXT: .LBB142_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB142_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] ; GCN3-NEXT: s_mov_b64 s[10:11], 0 ; GCN3-NEXT: .LBB142_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3] ; GCN3-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v6 ; GCN3-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN3-NEXT: v_mov_b32_e32 v7, v5 ; GCN3-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; GCN3-NEXT: v_mov_b32_e32 v6, v4 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[10:11] ; GCN3-NEXT: s_cbranch_execnz .LBB142_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[10:11] ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN3-NEXT: s_cbranch_execz .LBB142_2 ; GCN3-NEXT: .LBB142_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v0 ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v1, s[6:7] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst ret void } define i64 @flat_atomic_udec_wrap_i64_ret(ptr %ptr, i64 %in) { ; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB143_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: flat_load_dword v5, v[4:5] ; GCN1-NEXT: flat_load_dword v4, v[0:1] ; GCN1-NEXT: s_mov_b64 s[10:11], 0 ; GCN1-NEXT: .LBB143_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v7, v5 ; GCN1-NEXT: v_mov_b32_e32 v6, v4 ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3] ; GCN1-NEXT: v_add_i32_e64 v4, s[6:7], -1, v6 ; GCN1-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN1-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[10:11] ; GCN1-NEXT: s_cbranch_execnz .LBB143_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[10:11] ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: .LBB143_4: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN1-NEXT: s_cbranch_execz .LBB143_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e64 v6, s[6:7], -1, v4 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[4:5], v[2:3] ; GCN1-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v5, s[6:7] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN1-NEXT: .LBB143_6: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN1-NEXT: v_mov_b32_e32 v0, v4 ; GCN1-NEXT: v_mov_b32_e32 v1, v5 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB143_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: flat_load_dword v5, v[4:5] ; GCN2-NEXT: flat_load_dword v4, v[0:1] ; GCN2-NEXT: s_mov_b64 s[10:11], 0 ; GCN2-NEXT: .LBB143_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v7, v5 ; GCN2-NEXT: v_mov_b32_e32 v6, v4 ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3] ; GCN2-NEXT: v_add_u32_e64 v4, s[6:7], -1, v6 ; GCN2-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] ; GCN2-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[10:11] ; GCN2-NEXT: s_cbranch_execnz .LBB143_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[10:11] ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: .LBB143_4: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN2-NEXT: s_cbranch_execz .LBB143_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e64 v6, s[6:7], -1, v4 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[4:5], v[2:3] ; GCN2-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v5, s[6:7] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen ; GCN2-NEXT: .LBB143_6: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN2-NEXT: v_mov_b32_e32 v0, v4 ; GCN2-NEXT: v_mov_b32_e32 v1, v5 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v5, v1 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_mov_b32_e32 v4, v0 ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB143_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN3-NEXT: s_cbranch_execnz .LBB143_6 ; GCN3-NEXT: .LBB143_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB143_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[10:11], 0 ; GCN3-NEXT: .LBB143_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3] ; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v8 ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v9, s[6:7] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[10:11] ; GCN3-NEXT: s_cbranch_execnz .LBB143_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[10:11] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN3-NEXT: s_cbranch_execz .LBB143_2 ; GCN3-NEXT: .LBB143_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e64 v5, s[6:7], -1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN3-NEXT: v_addc_co_u32_e64 v6, s[6:7], -1, v1, s[6:7] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst ret i64 %result } define i64 @flat_atomic_udec_wrap_i64_ret_offset(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB144_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN1-NEXT: s_cbranch_execnz .LBB144_6 ; GCN1-NEXT: .LBB144_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB144_3: ; %atomicrmw.global ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[4:5] ; GCN1-NEXT: s_mov_b64 s[10:11], 0 ; GCN1-NEXT: .LBB144_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3] ; GCN1-NEXT: v_add_i32_e64 v0, s[6:7], -1, v8 ; GCN1-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v9, s[6:7] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[10:11] ; GCN1-NEXT: s_cbranch_execnz .LBB144_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[10:11] ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN1-NEXT: s_cbranch_execz .LBB144_2 ; GCN1-NEXT: .LBB144_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e64 v6, s[6:7], -1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN1-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB144_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN2-NEXT: s_cbranch_execnz .LBB144_6 ; GCN2-NEXT: .LBB144_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB144_3: ; %atomicrmw.global ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[4:5] ; GCN2-NEXT: s_mov_b64 s[10:11], 0 ; GCN2-NEXT: .LBB144_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3] ; GCN2-NEXT: v_add_u32_e64 v0, s[6:7], -1, v8 ; GCN2-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v9, s[6:7] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[10:11] ; GCN2-NEXT: s_cbranch_execnz .LBB144_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[10:11] ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN2-NEXT: s_cbranch_execz .LBB144_2 ; GCN2-NEXT: .LBB144_6: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e64 v6, s[6:7], -1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN2-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB144_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN3-NEXT: s_cbranch_execnz .LBB144_6 ; GCN3-NEXT: .LBB144_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB144_3: ; %atomicrmw.global ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] ; GCN3-NEXT: s_mov_b64 s[10:11], 0 ; GCN3-NEXT: .LBB144_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3] ; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v8 ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v9, s[6:7] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[10:11] ; GCN3-NEXT: s_cbranch_execnz .LBB144_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[10:11] ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9] ; GCN3-NEXT: s_cbranch_execz .LBB144_2 ; GCN3-NEXT: .LBB144_6: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e64 v5, s[6:7], -1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN3-NEXT: v_addc_co_u32_e64 v6, s[6:7], -1, v1, s[6:7] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst ret i64 %result } define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB145_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccnz .LBB145_6 ; GCN1-NEXT: .LBB145_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB145_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v4, s4 ; GCN1-NEXT: v_mov_b32_e32 v5, s5 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[38:39], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s7 ; GCN1-NEXT: v_mov_b32_e32 v7, s6 ; GCN1-NEXT: .LBB145_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] ; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3] ; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v2 ; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[38:39], vcc, s[38:39] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[38:39] ; GCN1-NEXT: s_cbranch_execnz .LBB145_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[38:39] ; GCN1-NEXT: s_branch .LBB145_2 ; GCN1-NEXT: .LBB145_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] ; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v0 ; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v1, s[36:37] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB145_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccnz .LBB145_6 ; GCN2-NEXT: .LBB145_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB145_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v4, s4 ; GCN2-NEXT: v_mov_b32_e32 v5, s5 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[38:39], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s7 ; GCN2-NEXT: v_mov_b32_e32 v7, s6 ; GCN2-NEXT: .LBB145_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] ; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3] ; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v2 ; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[38:39], vcc, s[38:39] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[38:39] ; GCN2-NEXT: s_cbranch_execnz .LBB145_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[38:39] ; GCN2-NEXT: s_branch .LBB145_2 ; GCN2-NEXT: .LBB145_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] ; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v0 ; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v1, s[36:37] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB145_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccnz .LBB145_6 ; GCN3-NEXT: .LBB145_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB145_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s4 ; GCN3-NEXT: v_mov_b32_e32 v5, s5 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[38:39], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s7 ; GCN3-NEXT: v_mov_b32_e32 v7, s6 ; GCN3-NEXT: .LBB145_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] ; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3] ; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v2 ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[38:39], vcc, s[38:39] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[38:39] ; GCN3-NEXT: s_cbranch_execnz .LBB145_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[38:39] ; GCN3-NEXT: s_branch .LBB145_2 ; GCN3-NEXT: .LBB145_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] ; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v0 ; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v1, s[36:37] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst ret void } define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s38, s4, 32 ; GCN1-NEXT: s_addc_u32 s39, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s39, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_mov_b64 s[34:35], -1 ; GCN1-NEXT: s_cbranch_vccnz .LBB146_3 ; GCN1-NEXT: ; %bb.1: ; %Flow3 ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccnz .LBB146_6 ; GCN1-NEXT: .LBB146_2: ; %atomicrmw.phi ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB146_3: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s38, 4 ; GCN1-NEXT: s_addc_u32 s35, s39, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v4, s38 ; GCN1-NEXT: v_mov_b32_e32 v5, s39 ; GCN1-NEXT: flat_load_dword v3, v[0:1] ; GCN1-NEXT: flat_load_dword v2, v[4:5] ; GCN1-NEXT: s_mov_b64 s[40:41], 0 ; GCN1-NEXT: v_mov_b32_e32 v6, s7 ; GCN1-NEXT: v_mov_b32_e32 v7, s6 ; GCN1-NEXT: .LBB146_4: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] ; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3] ; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v2 ; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN1-NEXT: v_mov_b32_e32 v3, v1 ; GCN1-NEXT: s_or_b64 s[40:41], vcc, s[40:41] ; GCN1-NEXT: v_mov_b32_e32 v2, v0 ; GCN1-NEXT: s_andn2_b64 exec, exec, s[40:41] ; GCN1-NEXT: s_cbranch_execnz .LBB146_4 ; GCN1-NEXT: ; %bb.5: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[40:41] ; GCN1-NEXT: s_branch .LBB146_2 ; GCN1-NEXT: .LBB146_6: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[38:39], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s38, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] ; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v0 ; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v1, s[36:37] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s38, s4, 32 ; GCN2-NEXT: s_addc_u32 s39, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s39, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_mov_b64 s[34:35], -1 ; GCN2-NEXT: s_cbranch_vccnz .LBB146_3 ; GCN2-NEXT: ; %bb.1: ; %Flow3 ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccnz .LBB146_6 ; GCN2-NEXT: .LBB146_2: ; %atomicrmw.phi ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB146_3: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s38, 4 ; GCN2-NEXT: s_addc_u32 s35, s39, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v4, s38 ; GCN2-NEXT: v_mov_b32_e32 v5, s39 ; GCN2-NEXT: flat_load_dword v3, v[0:1] ; GCN2-NEXT: flat_load_dword v2, v[4:5] ; GCN2-NEXT: s_mov_b64 s[40:41], 0 ; GCN2-NEXT: v_mov_b32_e32 v6, s7 ; GCN2-NEXT: v_mov_b32_e32 v7, s6 ; GCN2-NEXT: .LBB146_4: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] ; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3] ; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v2 ; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN2-NEXT: v_mov_b32_e32 v3, v1 ; GCN2-NEXT: s_or_b64 s[40:41], vcc, s[40:41] ; GCN2-NEXT: v_mov_b32_e32 v2, v0 ; GCN2-NEXT: s_andn2_b64 exec, exec, s[40:41] ; GCN2-NEXT: s_cbranch_execnz .LBB146_4 ; GCN2-NEXT: ; %bb.5: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[40:41] ; GCN2-NEXT: s_branch .LBB146_2 ; GCN2-NEXT: .LBB146_6: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[38:39], 0 ; GCN2-NEXT: s_cselect_b32 s34, s38, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] ; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v0 ; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v1, s[36:37] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s38, s4, 32 ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_addc_u32 s39, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s39, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_mov_b64 s[34:35], -1 ; GCN3-NEXT: s_cbranch_vccnz .LBB146_3 ; GCN3-NEXT: ; %bb.1: ; %Flow3 ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccnz .LBB146_6 ; GCN3-NEXT: .LBB146_2: ; %atomicrmw.phi ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB146_3: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v4, s38 ; GCN3-NEXT: v_mov_b32_e32 v5, s39 ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] ; GCN3-NEXT: s_mov_b64 s[40:41], 0 ; GCN3-NEXT: v_mov_b32_e32 v6, s7 ; GCN3-NEXT: v_mov_b32_e32 v7, s6 ; GCN3-NEXT: .LBB146_4: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] ; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3] ; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v2 ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GCN3-NEXT: v_mov_b32_e32 v3, v1 ; GCN3-NEXT: s_or_b64 s[40:41], vcc, s[40:41] ; GCN3-NEXT: v_mov_b32_e32 v2, v0 ; GCN3-NEXT: s_andn2_b64 exec, exec, s[40:41] ; GCN3-NEXT: s_cbranch_execnz .LBB146_4 ; GCN3-NEXT: ; %bb.5: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[40:41] ; GCN3-NEXT: s_branch .LBB146_2 ; GCN3-NEXT: .LBB146_6: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[38:39], 0 ; GCN3-NEXT: s_cselect_b32 s34, s38, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] ; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v0 ; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v1, s[36:37] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst ret void } define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB147_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s4, 4 ; GCN1-NEXT: s_addc_u32 s35, s5, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v2, s4 ; GCN1-NEXT: v_mov_b32_e32 v3, s5 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[38:39], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: .LBB147_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] ; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9] ; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v8 ; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[38:39], vcc, s[38:39] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[38:39] ; GCN1-NEXT: s_cbranch_execnz .LBB147_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[38:39] ; GCN1-NEXT: s_branch .LBB147_6 ; GCN1-NEXT: .LBB147_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB147_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e64 v6, s[36:37], -1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] ; GCN1-NEXT: v_addc_u32_e64 v7, s[36:37], -1, v1, s[36:37] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB147_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB147_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s4, 4 ; GCN2-NEXT: s_addc_u32 s35, s5, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v2, s4 ; GCN2-NEXT: v_mov_b32_e32 v3, s5 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[38:39], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: .LBB147_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] ; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9] ; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v8 ; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[38:39], vcc, s[38:39] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[38:39] ; GCN2-NEXT: s_cbranch_execnz .LBB147_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[38:39] ; GCN2-NEXT: s_branch .LBB147_6 ; GCN2-NEXT: .LBB147_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB147_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e64 v6, s[36:37], -1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] ; GCN2-NEXT: v_addc_u32_e64 v7, s[36:37], -1, v1, s[36:37] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB147_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB147_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s4 ; GCN3-NEXT: v_mov_b32_e32 v3, s5 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[38:39], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s7 ; GCN3-NEXT: v_mov_b32_e32 v5, s6 ; GCN3-NEXT: .LBB147_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] ; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9] ; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v8 ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v9, s[36:37] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[38:39], vcc, s[38:39] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[38:39] ; GCN3-NEXT: s_cbranch_execnz .LBB147_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[38:39] ; GCN3-NEXT: s_branch .LBB147_6 ; GCN3-NEXT: .LBB147_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB147_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e64 v5, s[36:37], -1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] ; GCN3-NEXT: v_addc_co_u32_e64 v6, s[36:37], -1, v1, s[36:37] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB147_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst ret i64 %result } define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { ; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN1-NEXT: s_add_u32 s38, s4, 32 ; GCN1-NEXT: s_addc_u32 s39, s5, 0 ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: s_cmp_eq_u32 s39, s34 ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN1-NEXT: s_cbranch_vccz .LBB148_4 ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN1-NEXT: s_add_u32 s34, s38, 4 ; GCN1-NEXT: s_addc_u32 s35, s39, 0 ; GCN1-NEXT: v_mov_b32_e32 v0, s34 ; GCN1-NEXT: v_mov_b32_e32 v1, s35 ; GCN1-NEXT: v_mov_b32_e32 v2, s38 ; GCN1-NEXT: v_mov_b32_e32 v3, s39 ; GCN1-NEXT: flat_load_dword v1, v[0:1] ; GCN1-NEXT: flat_load_dword v0, v[2:3] ; GCN1-NEXT: s_mov_b64 s[40:41], 0 ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: .LBB148_2: ; %atomicrmw.start ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: v_mov_b32_e32 v9, v1 ; GCN1-NEXT: v_mov_b32_e32 v8, v0 ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] ; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9] ; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v8 ; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN1-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN1-NEXT: s_or_b64 s[40:41], vcc, s[40:41] ; GCN1-NEXT: s_andn2_b64 exec, exec, s[40:41] ; GCN1-NEXT: s_cbranch_execnz .LBB148_2 ; GCN1-NEXT: ; %bb.3: ; %Flow ; GCN1-NEXT: s_or_b64 exec, exec, s[40:41] ; GCN1-NEXT: s_branch .LBB148_6 ; GCN1-NEXT: .LBB148_4: ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_cbranch_execz .LBB148_6 ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[38:39], 0 ; GCN1-NEXT: v_mov_b32_e32 v5, s6 ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec ; GCN1-NEXT: s_cselect_b32 s34, s38, -1 ; GCN1-NEXT: v_mov_b32_e32 v2, s34 ; GCN1-NEXT: s_add_i32 s34, s34, 4 ; GCN1-NEXT: v_mov_b32_e32 v3, s34 ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN1-NEXT: v_mov_b32_e32 v4, s7 ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e64 v6, s[36:37], -1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] ; GCN1-NEXT: v_addc_u32_e64 v7, s[36:37], -1, v1, s[36:37] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN1-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN1-NEXT: .LBB148_6: ; %atomicrmw.phi ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 ; GCN2-NEXT: s_add_u32 s38, s4, 32 ; GCN2-NEXT: s_addc_u32 s39, s5, 0 ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: s_cmp_eq_u32 s39, s34 ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN2-NEXT: s_cbranch_vccz .LBB148_4 ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN2-NEXT: s_add_u32 s34, s38, 4 ; GCN2-NEXT: s_addc_u32 s35, s39, 0 ; GCN2-NEXT: v_mov_b32_e32 v0, s34 ; GCN2-NEXT: v_mov_b32_e32 v1, s35 ; GCN2-NEXT: v_mov_b32_e32 v2, s38 ; GCN2-NEXT: v_mov_b32_e32 v3, s39 ; GCN2-NEXT: flat_load_dword v1, v[0:1] ; GCN2-NEXT: flat_load_dword v0, v[2:3] ; GCN2-NEXT: s_mov_b64 s[40:41], 0 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: .LBB148_2: ; %atomicrmw.start ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: v_mov_b32_e32 v9, v1 ; GCN2-NEXT: v_mov_b32_e32 v8, v0 ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] ; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9] ; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v8 ; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN2-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN2-NEXT: s_or_b64 s[40:41], vcc, s[40:41] ; GCN2-NEXT: s_andn2_b64 exec, exec, s[40:41] ; GCN2-NEXT: s_cbranch_execnz .LBB148_2 ; GCN2-NEXT: ; %bb.3: ; %Flow ; GCN2-NEXT: s_or_b64 exec, exec, s[40:41] ; GCN2-NEXT: s_branch .LBB148_6 ; GCN2-NEXT: .LBB148_4: ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_cbranch_execz .LBB148_6 ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN2-NEXT: s_cmp_lg_u64 s[38:39], 0 ; GCN2-NEXT: s_cselect_b32 s34, s38, -1 ; GCN2-NEXT: v_mov_b32_e32 v2, s34 ; GCN2-NEXT: s_add_i32 s34, s34, 4 ; GCN2-NEXT: v_mov_b32_e32 v3, s34 ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen ; GCN2-NEXT: v_mov_b32_e32 v5, s6 ; GCN2-NEXT: v_mov_b32_e32 v4, s7 ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e64 v6, s[36:37], -1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] ; GCN2-NEXT: v_addc_u32_e64 v7, s[36:37], -1, v1, s[36:37] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN2-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen ; GCN2-NEXT: .LBB148_6: ; %atomicrmw.phi ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: s_add_u32 s38, s4, 32 ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base ; GCN3-NEXT: s_addc_u32 s39, s5, 0 ; GCN3-NEXT: s_cmp_eq_u32 s39, s35 ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] ; GCN3-NEXT: s_cbranch_vccz .LBB148_4 ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global ; GCN3-NEXT: v_mov_b32_e32 v2, s38 ; GCN3-NEXT: v_mov_b32_e32 v3, s39 ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] ; GCN3-NEXT: s_mov_b64 s[40:41], 0 ; GCN3-NEXT: v_mov_b32_e32 v4, s7 ; GCN3-NEXT: v_mov_b32_e32 v5, s6 ; GCN3-NEXT: .LBB148_2: ; %atomicrmw.start ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_mov_b32_e32 v9, v1 ; GCN3-NEXT: v_mov_b32_e32 v8, v0 ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] ; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9] ; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v8 ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v9, s[36:37] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN3-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] ; GCN3-NEXT: s_or_b64 s[40:41], vcc, s[40:41] ; GCN3-NEXT: s_andn2_b64 exec, exec, s[40:41] ; GCN3-NEXT: s_cbranch_execnz .LBB148_2 ; GCN3-NEXT: ; %bb.3: ; %Flow ; GCN3-NEXT: s_or_b64 exec, exec, s[40:41] ; GCN3-NEXT: s_branch .LBB148_6 ; GCN3-NEXT: .LBB148_4: ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_cbranch_execz .LBB148_6 ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private ; GCN3-NEXT: s_cmp_lg_u64 s[38:39], 0 ; GCN3-NEXT: s_cselect_b32 s34, s38, -1 ; GCN3-NEXT: v_mov_b32_e32 v2, s34 ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: v_mov_b32_e32 v4, s6 ; GCN3-NEXT: v_mov_b32_e32 v3, s7 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e64 v5, s[36:37], -1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] ; GCN3-NEXT: v_addc_co_u32_e64 v6, s[36:37], -1, v1, s[36:37] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35] ; GCN3-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 ; GCN3-NEXT: .LBB148_6: ; %atomicrmw.phi ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst ret i64 %result } define void @flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB149_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB149_4 ; GCN1-NEXT: .LBB149_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB149_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB149_2 ; GCN1-NEXT: .LBB149_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN1-NEXT: v_add_i32_e64 v0, s[6:7], -1, v0 ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN1-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7] ; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB149_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB149_4 ; GCN2-NEXT: .LBB149_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB149_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB149_2 ; GCN2-NEXT: .LBB149_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN2-NEXT: v_add_u32_e64 v0, s[6:7], -1, v0 ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN2-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7] ; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB149_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB149_4 ; GCN3-NEXT: .LBB149_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB149_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB149_2 ; GCN3-NEXT: .LBB149_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v0 ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v1, s[6:7] ; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret void } define i64 @flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { ; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory: ; GCN1: ; %bb.0: ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN1-NEXT: s_waitcnt lgkmcnt(0) ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB150_3 ; GCN1-NEXT: ; %bb.1: ; %Flow ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] ; GCN1-NEXT: s_cbranch_execnz .LBB150_4 ; GCN1-NEXT: .LBB150_2: ; %atomicrmw.phi ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN1-NEXT: s_setpc_b64 s[30:31] ; GCN1-NEXT: .LBB150_3: ; %atomicrmw.global ; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN1-NEXT: buffer_wbinvl1_vol ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] ; GCN1-NEXT: s_cbranch_execz .LBB150_2 ; GCN1-NEXT: .LBB150_4: ; %atomicrmw.private ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN1-NEXT: s_waitcnt vmcnt(1) ; GCN1-NEXT: v_add_i32_e64 v6, s[6:7], -1, v0 ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN1-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7] ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN1-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc ; GCN1-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN1-NEXT: s_waitcnt vmcnt(0) ; GCN1-NEXT: s_setpc_b64 s[30:31] ; ; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory: ; GCN2: ; %bb.0: ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc ; GCN2-NEXT: s_waitcnt lgkmcnt(0) ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB150_3 ; GCN2-NEXT: ; %bb.1: ; %Flow ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] ; GCN2-NEXT: s_cbranch_execnz .LBB150_4 ; GCN2-NEXT: .LBB150_2: ; %atomicrmw.phi ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN2-NEXT: s_setpc_b64 s[30:31] ; GCN2-NEXT: .LBB150_3: ; %atomicrmw.global ; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN2-NEXT: buffer_wbinvl1_vol ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] ; GCN2-NEXT: s_cbranch_execz .LBB150_2 ; GCN2-NEXT: .LBB150_4: ; %atomicrmw.private ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen ; GCN2-NEXT: s_waitcnt vmcnt(1) ; GCN2-NEXT: v_add_u32_e64 v6, s[6:7], -1, v0 ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN2-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7] ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN2-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc ; GCN2-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN2-NEXT: s_waitcnt vmcnt(0) ; GCN2-NEXT: s_setpc_b64 s[30:31] ; ; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory: ; GCN3: ; %bb.0: ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB150_3 ; GCN3-NEXT: ; %bb.1: ; %Flow ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] ; GCN3-NEXT: s_cbranch_execnz .LBB150_4 ; GCN3-NEXT: .LBB150_2: ; %atomicrmw.phi ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN3-NEXT: s_setpc_b64 s[30:31] ; GCN3-NEXT: .LBB150_3: ; %atomicrmw.global ; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN3-NEXT: buffer_wbinvl1_vol ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] ; GCN3-NEXT: s_cbranch_execz .LBB150_2 ; GCN3-NEXT: .LBB150_4: ; %atomicrmw.private ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_waitcnt vmcnt(1) ; GCN3-NEXT: v_add_co_u32_e64 v5, s[6:7], -1, v0 ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] ; GCN3-NEXT: v_addc_co_u32_e64 v6, s[6:7], -1, v1, s[6:7] ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] ; GCN3-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc ; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN3-NEXT: s_waitcnt vmcnt(0) ; GCN3-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr i64, ptr %out, i64 4 %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 ret i64 %result } !0 = !{}