diff options
author | Stanislav Mekhanoshin <rampitec@users.noreply.github.com> | 2023-12-13 10:49:36 -0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-12-13 10:49:36 -0800 |
commit | c6ecbcb48b9569e6938f4382c67e4b9ac1bf3bdd (patch) | |
tree | 367fe6bf2742f2b658a080982556c06780daf491 | |
parent | a5ffabce98a4b2e9d69009fa3e60f2b154100860 (diff) | |
download | llvm-c6ecbcb48b9569e6938f4382c67e4b9ac1bf3bdd.zip llvm-c6ecbcb48b9569e6938f4382c67e4b9ac1bf3bdd.tar.gz llvm-c6ecbcb48b9569e6938f4382c67e4b9ac1bf3bdd.tar.bz2 |
[AMDGPU] Fix no waitcnt produced between LDS DMA and ds_read on gfx10 (#75245)
BUFFER_LOAD_DWORD_LDS was incorrectly touching vscnt instead of the
vmcnt. This is VMEM load and DS store, so it shall use vmcnt.
-rw-r--r-- | llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 19 | ||||
-rw-r--r-- | llvm/lib/Target/AMDGPU/SIInstrInfo.h | 12 | ||||
-rw-r--r-- | llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll | 11 |
3 files changed, 23 insertions, 19 deletions
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp index 13426e7..c16583f 100644 --- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp +++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp @@ -452,7 +452,9 @@ public: // FLAT instruction. WaitEventType getVmemWaitEventType(const MachineInstr &Inst) const { assert(SIInstrInfo::isVMEM(Inst) || SIInstrInfo::isFLAT(Inst)); - if (!ST->hasVscnt()) + // LDS DMA loads are also stores, but on the LDS side. On the VMEM side + // these should use VM_CNT. + if (!ST->hasVscnt() || SIInstrInfo::mayWriteLDSThroughDMA(Inst)) return VMEM_ACCESS; if (Inst.mayStore() && !SIInstrInfo::isAtomicRet(Inst)) { // FLAT and SCRATCH instructions may access scratch. Other VMEM @@ -544,14 +546,6 @@ void WaitcntBrackets::setExpScore(const MachineInstr *MI, } } -// MUBUF and FLAT LDS DMA operations need a wait on vmcnt before LDS written -// can be accessed. A load from LDS to VMEM does not need a wait. -static bool mayWriteLDSThroughDMA(const MachineInstr &MI) { - return SIInstrInfo::isVALU(MI) && - (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI)) && - MI.getOpcode() != AMDGPU::BUFFER_STORE_LDS_DWORD; -} - void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII, const SIRegisterInfo *TRI, const MachineRegisterInfo *MRI, @@ -703,7 +697,10 @@ void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII, setRegScore(RegNo, T, CurrScore); } } - if (Inst.mayStore() && (TII->isDS(Inst) || mayWriteLDSThroughDMA(Inst))) { + if (Inst.mayStore() && + (TII->isDS(Inst) || TII->mayWriteLDSThroughDMA(Inst))) { + // MUBUF and FLAT LDS DMA operations need a wait on vmcnt before LDS + // written can be accessed. A load from LDS to VMEM does not need a wait. setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS, T, CurrScore); } } @@ -1178,7 +1175,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI, if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::FLAT_ADDRESS) continue; // No need to wait before load from VMEM to LDS. - if (mayWriteLDSThroughDMA(MI)) + if (TII->mayWriteLDSThroughDMA(MI)) continue; unsigned RegNo = SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS; // VM_CNT is only relevant to vgpr or LDS. diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h index e794d8c..32a81d6 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -546,6 +546,14 @@ public: return get(Opcode).TSFlags & SIInstrFlags::DS; } + static bool isLDSDMA(const MachineInstr &MI) { + return isVALU(MI) && (isMUBUF(MI) || isFLAT(MI)); + } + + bool isLDSDMA(uint16_t Opcode) { + return isVALU(Opcode) && (isMUBUF(Opcode) || isFLAT(Opcode)); + } + static bool isGWS(const MachineInstr &MI) { return MI.getDesc().TSFlags & SIInstrFlags::GWS; } @@ -667,6 +675,10 @@ public: SIInstrFlags::IsAtomicNoRet); } + static bool mayWriteLDSThroughDMA(const MachineInstr &MI) { + return isLDSDMA(MI) && MI.getOpcode() != AMDGPU::BUFFER_STORE_LDS_DWORD; + } + static bool isWQM(const MachineInstr &MI) { return MI.getDesc().TSFlags & SIInstrFlags::WQM; } diff --git a/llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll b/llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll index 688f031..58c6406 100644 --- a/llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll +++ b/llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll @@ -9,15 +9,10 @@ declare void @llvm.amdgcn.global.load.lds(ptr addrspace(1) nocapture %gptr, ptr ; FIXME: vmcnt(0) is too strong, it shall use vmcnt(2) before the first ; ds_read_b32 and vmcnt(0) before the second. -; FIXME: GFX10 does not get a waitcount at all. ; GCN-LABEL: {{^}}buffer_load_lds_dword_2_arrays: ; GCN-COUNT-4: buffer_load_dword -; GFX9: s_waitcnt vmcnt(0) - -; FIXME: -; GFX10-NOT: s_waitcnt - +; GCN: s_waitcnt vmcnt(0) ; GCN: ds_read_b32 ; FIXME: @@ -49,9 +44,9 @@ main_body: ; GFX9: s_waitcnt vmcnt(0) ; GFX9-COUNT-2: ds_read_b32 -; FIXME: -; GFX10-NOT: s_waitcnt +; FIXME: can be vmcnt(2) +; GFX10: s_waitcnt vmcnt(0) ; GFX10: ds_read_b32 ; FIXME: |