aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Target/AMDGPU
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Target/AMDGPU')
-rw-r--r--llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp8
-rw-r--r--llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp5
-rw-r--r--llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h3
-rw-r--r--llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp16
-rw-r--r--llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp3
-rw-r--r--llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h3
-rw-r--r--llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp16
-rw-r--r--llvm/lib/Target/AMDGPU/DSDIRInstructions.td191
-rw-r--r--llvm/lib/Target/AMDGPU/LDSDIRInstructions.td116
-rw-r--r--llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp14
-rw-r--r--llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h4
-rw-r--r--llvm/lib/Target/AMDGPU/SIInstrInfo.td2
-rw-r--r--llvm/lib/Target/AMDGPU/SIInstructions.td2
-rw-r--r--llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp4
-rw-r--r--llvm/lib/Target/AMDGPU/SMInstructions.td25
15 files changed, 276 insertions, 136 deletions
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 88ef4b5..ad8dcda 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -2764,7 +2764,9 @@ static bool isConstant(const MachineInstr &MI) {
void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load,
const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const {
- const MachineInstr *PtrMI = MRI.getUniqueVRegDef(Load.getOperand(1).getReg());
+ unsigned OpNo = Load.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1;
+ const MachineInstr *PtrMI =
+ MRI.getUniqueVRegDef(Load.getOperand(OpNo).getReg());
assert(PtrMI);
@@ -2817,6 +2819,10 @@ bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const {
if (MMO->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
return true;
+ if (MI.getOpcode() == AMDGPU::G_PREFETCH)
+ return RBI.getRegBank(MI.getOperand(0).getReg(), *MRI, TRI)->getID() ==
+ AMDGPU::SGPRRegBankID;
+
const Instruction *I = dyn_cast<Instruction>(Ptr);
return I && I->getMetadata("amdgpu.uniform");
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index fbee288..dfbe5c7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1996,8 +1996,9 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
verify(*ST.getInstrInfo());
}
-bool AMDGPULegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
- MachineInstr &MI) const {
+bool AMDGPULegalizerInfo::legalizeCustom(
+ LegalizerHelper &Helper, MachineInstr &MI,
+ LostDebugLocObserver &LocObserver) const {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 855fa0d..1fa0648 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -35,7 +35,8 @@ public:
AMDGPULegalizerInfo(const GCNSubtarget &ST,
const GCNTargetMachine &TM);
- bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI) const override;
+ bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
+ LostDebugLocObserver &LocObserver) const override;
Register getSegmentAperture(unsigned AddrSpace,
MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index fba0604..92182ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3263,17 +3263,19 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
MI.eraseFromParent();
return;
}
- unsigned PtrBank =
- getRegBankID(MI.getOperand(0).getReg(), MRI, AMDGPU::SGPRRegBankID);
+ Register PtrReg = MI.getOperand(0).getReg();
+ unsigned PtrBank = getRegBankID(PtrReg, MRI, AMDGPU::SGPRRegBankID);
if (PtrBank == AMDGPU::VGPRRegBankID) {
MI.eraseFromParent();
return;
}
- // FIXME: There is currently no support for prefetch in global isel.
- // There is no node equivalence and what's worse there is no MMO produced
- // for a prefetch on global isel path.
- // Prefetch does not affect execution so erase it for now.
- MI.eraseFromParent();
+ unsigned AS = MRI.getType(PtrReg).getAddressSpace();
+ if (!AMDGPU::isFlatGlobalAddrSpace(AS) &&
+ AS != AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
+ MI.eraseFromParent();
+ return;
+ }
+ applyDefaultMapping(OpdMapper);
return;
}
default:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index fdc2077..0f3bb3e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -620,7 +620,8 @@ void AMDGPUTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) {
AAM.registerFunctionAnalysis<AMDGPUAA>();
}
-void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
+void AMDGPUTargetMachine::registerPassBuilderCallbacks(
+ PassBuilder &PB, bool PopulateClassToPassNames) {
PB.registerPipelineParsingCallback(
[this](StringRef PassName, ModulePassManager &PM,
ArrayRef<PassBuilder::PipelineElement>) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 9051a61..99c9db3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -51,7 +51,8 @@ public:
return TLOF.get();
}
- void registerPassBuilderCallbacks(PassBuilder &PB) override;
+ void registerPassBuilderCallbacks(PassBuilder &PB,
+ bool PopulateClassToPassNames) override;
void registerDefaultAliasAnalyses(AAManager &) override;
/// Get the integer value of a null pointer in the given address space.
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index abd7e91..5f2b7c0 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -166,6 +166,8 @@ public:
ImmTyEndpgm,
ImmTyWaitVDST,
ImmTyWaitEXP,
+ ImmTyWaitVAVDst,
+ ImmTyWaitVMVSrc,
};
// Immediate operand kind.
@@ -909,6 +911,8 @@ public:
bool isEndpgm() const;
bool isWaitVDST() const;
bool isWaitEXP() const;
+ bool isWaitVAVDst() const;
+ bool isWaitVMVSrc() const;
auto getPredicate(std::function<bool(const AMDGPUOperand &Op)> P) const {
return std::bind(P, *this);
@@ -1029,6 +1033,7 @@ public:
}
static void printImmTy(raw_ostream& OS, ImmTy Type) {
+ // clang-format off
switch (Type) {
case ImmTyNone: OS << "None"; break;
case ImmTyGDS: OS << "GDS"; break;
@@ -1086,7 +1091,10 @@ public:
case ImmTyEndpgm: OS << "Endpgm"; break;
case ImmTyWaitVDST: OS << "WaitVDST"; break;
case ImmTyWaitEXP: OS << "WaitEXP"; break;
+ case ImmTyWaitVAVDst: OS << "WaitVAVDst"; break;
+ case ImmTyWaitVMVSrc: OS << "WaitVMVSrc"; break;
}
+ // clang-format on
}
void print(raw_ostream &OS) const override {
@@ -9192,6 +9200,14 @@ bool AMDGPUOperand::isWaitVDST() const {
return isImmTy(ImmTyWaitVDST) && isUInt<4>(getImm());
}
+bool AMDGPUOperand::isWaitVAVDst() const {
+ return isImmTy(ImmTyWaitVAVDst) && isUInt<4>(getImm());
+}
+
+bool AMDGPUOperand::isWaitVMVSrc() const {
+ return isImmTy(ImmTyWaitVMVSrc) && isUInt<1>(getImm());
+}
+
//===----------------------------------------------------------------------===//
// VINTERP
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/DSDIRInstructions.td b/llvm/lib/Target/AMDGPU/DSDIRInstructions.td
new file mode 100644
index 0000000..54ef785
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/DSDIRInstructions.td
@@ -0,0 +1,191 @@
+//===-- DSDIRInstructions.td - LDS/VDS Direct Instruction Definitions -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// LDSDIR/VDSDIR encoding (LDSDIR is gfx11, VDSDIR is gfx12+)
+//===----------------------------------------------------------------------===//
+
+class LDSDIRe<bits<2> op, bit is_direct> : Enc32 {
+ // encoding fields
+ bits<2> attrchan;
+ bits<6> attr;
+ bits<4> waitvdst;
+ bits<8> vdst;
+
+ // encoding
+ let Inst{31-24} = 0xce; // encoding
+ let Inst{23-22} = 0x0; // reserved
+ let Inst{21-20} = op;
+ let Inst{19-16} = waitvdst;
+ let Inst{15-10} = !if(is_direct, ?, attr);
+ let Inst{9-8} = !if(is_direct, ?, attrchan);
+ let Inst{7-0} = vdst;
+}
+
+class VDSDIRe<bits<2> op, bit is_direct> : Enc32 {
+ // encoding fields
+ bits<2> attrchan;
+ bits<6> attr;
+ bits<4> waitvdst;
+ bits<8> vdst;
+ bits<1> waitvsrc;
+
+ // encoding
+ let Inst{31-24} = 0xce; // encoding
+ let Inst{23} = waitvsrc;
+ let Inst{22} = 0x0; // reserved
+ let Inst{21-20} = op;
+ let Inst{19-16} = waitvdst;
+ let Inst{15-10} = !if(is_direct, ?, attr);
+ let Inst{9-8} = !if(is_direct, ?, attrchan);
+ let Inst{7-0} = vdst;
+}
+
+//===----------------------------------------------------------------------===//
+// LDSDIR/VDSDIR Classes
+//===----------------------------------------------------------------------===//
+
+class LDSDIR_getIns<bit direct> {
+ dag ret = !if(direct,
+ (ins wait_vdst:$waitvdst),
+ (ins InterpAttr:$attr, InterpAttrChan:$attrchan, wait_vdst:$waitvdst)
+ );
+}
+
+class VDSDIR_getIns<bit direct> {
+ dag ret = !if(direct,
+ (ins wait_va_vdst:$waitvdst, wait_va_vsrc:$waitvsrc),
+ (ins InterpAttr:$attr, InterpAttrChan:$attrchan, wait_va_vdst:$waitvdst,
+ wait_va_vsrc:$waitvsrc)
+ );
+}
+
+class DSDIR_Common<string opName, string asm = "", dag ins, bit direct> :
+ InstSI<(outs VGPR_32:$vdst), ins, asm> {
+ let LDSDIR = 1;
+ let EXP_CNT = 1;
+
+ let hasSideEffects = 0;
+ let mayLoad = 1;
+ let mayStore = 0;
+
+ string Mnemonic = opName;
+ let UseNamedOperandTable = 1;
+
+ let Uses = [M0, EXEC];
+ let DisableWQM = 0;
+ let SchedRW = [WriteLDS];
+
+ bit is_direct;
+ let is_direct = direct;
+}
+
+class DSDIR_Pseudo<string opName, dag ins, bit direct> :
+ DSDIR_Common<opName, "", ins, direct>,
+ SIMCInstr<opName, SIEncodingFamily.NONE> {
+ let isPseudo = 1;
+ let isCodeGenOnly = 1;
+}
+
+class LDSDIR_getAsm<bit direct> {
+ string ret = !if(direct,
+ " $vdst$waitvdst",
+ " $vdst, $attr$attrchan$waitvdst"
+ );
+}
+
+class VDSDIR_getAsm<bit direct> {
+ string ret = !if(direct,
+ " $vdst$waitvdst$waitvsrc",
+ " $vdst, $attr$attrchan$waitvdst$waitvsrc"
+ );
+}
+
+class DSDIR_Real<DSDIR_Pseudo lds, dag ins, string asm, int subtarget> :
+ DSDIR_Common<lds.Mnemonic,
+ lds.Mnemonic # asm,
+ ins,
+ lds.is_direct>,
+ SIMCInstr <lds.Mnemonic, subtarget> {
+ let isPseudo = 0;
+ let isCodeGenOnly = 0;
+}
+
+//===----------------------------------------------------------------------===//
+// LDS/VDS Direct Instructions
+//===----------------------------------------------------------------------===//
+
+let SubtargetPredicate = isGFX11Only in {
+
+def LDS_DIRECT_LOAD : DSDIR_Pseudo<"lds_direct_load", LDSDIR_getIns<1>.ret, 1>;
+def LDS_PARAM_LOAD : DSDIR_Pseudo<"lds_param_load", LDSDIR_getIns<0>.ret, 0>;
+
+def : GCNPat <
+ (f32 (int_amdgcn_lds_direct_load M0)),
+ (LDS_DIRECT_LOAD 0)
+>;
+
+def : GCNPat <
+ (f32 (int_amdgcn_lds_param_load timm:$attrchan, timm:$attr, M0)),
+ (LDS_PARAM_LOAD timm:$attr, timm:$attrchan, 0)
+>;
+
+} // End SubtargetPredicate = isGFX11Only
+
+let SubtargetPredicate = isGFX12Plus in {
+
+def DS_DIRECT_LOAD : DSDIR_Pseudo<"ds_direct_load", VDSDIR_getIns<1>.ret, 1>;
+def DS_PARAM_LOAD : DSDIR_Pseudo<"ds_param_load", VDSDIR_getIns<0>.ret, 0>;
+
+def : GCNPat <
+ (f32 (int_amdgcn_lds_direct_load M0)),
+ (DS_DIRECT_LOAD 0, 1)
+>;
+
+def : GCNPat <
+ (f32 (int_amdgcn_lds_param_load timm:$attrchan, timm:$attr, M0)),
+ (DS_PARAM_LOAD timm:$attr, timm:$attrchan, 0, 1)
+>;
+
+} // End SubtargetPredicate = isGFX12Only
+
+//===----------------------------------------------------------------------===//
+// GFX11
+//===----------------------------------------------------------------------===//
+
+multiclass DSDIR_Real_gfx11<bits<2> op,
+ DSDIR_Pseudo lds = !cast<DSDIR_Pseudo>(NAME)> {
+ def _gfx11 : DSDIR_Real<lds, lds.InOperandList,
+ LDSDIR_getAsm<lds.is_direct>.ret,
+ SIEncodingFamily.GFX11>,
+ LDSDIRe<op, lds.is_direct> {
+ let AssemblerPredicate = isGFX11Only;
+ let DecoderNamespace = "GFX11";
+ }
+}
+
+defm LDS_PARAM_LOAD : DSDIR_Real_gfx11<0x0>;
+defm LDS_DIRECT_LOAD : DSDIR_Real_gfx11<0x1>;
+
+//===----------------------------------------------------------------------===//
+// GFX12+
+//===----------------------------------------------------------------------===//
+
+multiclass DSDIR_Real_gfx12<bits<2> op,
+ DSDIR_Pseudo lds = !cast<DSDIR_Pseudo>(NAME)> {
+ def _gfx12 : DSDIR_Real<lds, lds.InOperandList,
+ VDSDIR_getAsm<lds.is_direct>.ret,
+ SIEncodingFamily.GFX12>,
+ VDSDIRe<op, lds.is_direct> {
+ let AssemblerPredicate = isGFX12Plus;
+ let DecoderNamespace = "GFX12";
+ }
+}
+
+defm DS_PARAM_LOAD : DSDIR_Real_gfx12<0x0>;
+defm DS_DIRECT_LOAD : DSDIR_Real_gfx12<0x1>;
diff --git a/llvm/lib/Target/AMDGPU/LDSDIRInstructions.td b/llvm/lib/Target/AMDGPU/LDSDIRInstructions.td
deleted file mode 100644
index 4956a15..0000000
--- a/llvm/lib/Target/AMDGPU/LDSDIRInstructions.td
+++ /dev/null
@@ -1,116 +0,0 @@
-//===-- LDSDIRInstructions.td - LDS Direct Instruction Definitions --------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-//===----------------------------------------------------------------------===//
-// LDSDIR encoding
-//===----------------------------------------------------------------------===//
-
-class LDSDIRe<bits<2> op, bit is_direct> : Enc32 {
- // encoding fields
- bits<2> attrchan;
- bits<6> attr;
- bits<4> waitvdst;
- bits<8> vdst;
-
- // encoding
- let Inst{31-24} = 0xce; // encoding
- let Inst{23-22} = 0x0; // reserved
- let Inst{21-20} = op;
- let Inst{19-16} = waitvdst;
- let Inst{15-10} = !if(is_direct, ?, attr);
- let Inst{9-8} = !if(is_direct, ?, attrchan);
- let Inst{7-0} = vdst;
-}
-
-//===----------------------------------------------------------------------===//
-// LDSDIR Classes
-//===----------------------------------------------------------------------===//
-
-class LDSDIR_getIns<bit direct> {
- dag ret = !if(direct,
- (ins wait_vdst:$waitvdst),
- (ins InterpAttr:$attr, InterpAttrChan:$attrchan, wait_vdst:$waitvdst)
- );
-}
-
-class LDSDIR_Common<string opName, string asm = "", bit direct> : InstSI<
- (outs VGPR_32:$vdst),
- LDSDIR_getIns<direct>.ret,
- asm> {
- let LDSDIR = 1;
- let EXP_CNT = 1;
-
- let hasSideEffects = 0;
- let mayLoad = 1;
- let mayStore = 0;
-
- string Mnemonic = opName;
- let UseNamedOperandTable = 1;
-
- let Uses = [M0, EXEC];
- let DisableWQM = 0;
- let SchedRW = [WriteLDS];
-
- bit is_direct;
- let is_direct = direct;
-}
-
-class LDSDIR_Pseudo<string opName, bit direct> :
- LDSDIR_Common<opName, "", direct>,
- SIMCInstr<opName, SIEncodingFamily.NONE> {
- let isPseudo = 1;
- let isCodeGenOnly = 1;
-}
-
-class LDSDIR_getAsm<bit direct> {
- string ret = !if(direct,
- " $vdst$waitvdst",
- " $vdst, $attr$attrchan$waitvdst"
- );
-}
-
-class LDSDIR_Real<bits<2> op, LDSDIR_Pseudo lds, int subtarget> :
- LDSDIR_Common<lds.Mnemonic,
- lds.Mnemonic # LDSDIR_getAsm<lds.is_direct>.ret,
- lds.is_direct>,
- SIMCInstr <lds.Mnemonic, subtarget>,
- LDSDIRe<op, lds.is_direct> {
- let isPseudo = 0;
- let isCodeGenOnly = 0;
-}
-
-//===----------------------------------------------------------------------===//
-// LDS Direct Instructions
-//===----------------------------------------------------------------------===//
-
-def LDS_DIRECT_LOAD : LDSDIR_Pseudo<"lds_direct_load", 1>;
-def LDS_PARAM_LOAD : LDSDIR_Pseudo<"lds_param_load", 0>;
-
-def : GCNPat <
- (f32 (int_amdgcn_lds_direct_load M0)),
- (LDS_DIRECT_LOAD 0)
->;
-
-def : GCNPat <
- (f32 (int_amdgcn_lds_param_load timm:$attrchan, timm:$attr, M0)),
- (LDS_PARAM_LOAD timm:$attr, timm:$attrchan, 0)
->;
-
-//===----------------------------------------------------------------------===//
-// GFX11+
-//===----------------------------------------------------------------------===//
-
-multiclass LDSDIR_Real_gfx11<bits<2> op, LDSDIR_Pseudo lds = !cast<LDSDIR_Pseudo>(NAME)> {
- def _gfx11 : LDSDIR_Real<op, lds, SIEncodingFamily.GFX11> {
- let AssemblerPredicate = isGFX11Plus;
- let DecoderNamespace = "GFX11";
- }
-}
-
-defm LDS_PARAM_LOAD : LDSDIR_Real_gfx11<0x0>;
-defm LDS_DIRECT_LOAD : LDSDIR_Real_gfx11<0x1>;
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index edc244d..ef1b85f 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -639,6 +639,20 @@ void AMDGPUInstPrinter::printWaitVDST(const MCInst *MI, unsigned OpNo,
printU4ImmDecOperand(MI, OpNo, O);
}
+void AMDGPUInstPrinter::printWaitVAVDst(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &STI,
+ raw_ostream &O) {
+ O << " wait_va_vdst:";
+ printU4ImmDecOperand(MI, OpNo, O);
+}
+
+void AMDGPUInstPrinter::printWaitVMVSrc(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &STI,
+ raw_ostream &O) {
+ O << " wait_vm_vsrc:";
+ printU4ImmDecOperand(MI, OpNo, O);
+}
+
void AMDGPUInstPrinter::printWaitEXP(const MCInst *MI, unsigned OpNo,
const MCSubtargetInfo &STI,
raw_ostream &O) {
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
index 95c26de..f2f985f 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
@@ -161,6 +161,10 @@ private:
raw_ostream &O);
void printWaitEXP(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
raw_ostream &O);
+ void printWaitVAVDst(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &STI, raw_ostream &O);
+ void printWaitVMVSrc(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &STI, raw_ostream &O);
void printExpSrcN(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
raw_ostream &O, unsigned N);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 173c877..50724fd 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -1144,6 +1144,8 @@ def exp_tgt : CustomOperand<i32, 0, "ExpTgt">;
def wait_vdst : NamedIntOperand<i8, "wait_vdst", "WaitVDST">;
def wait_exp : NamedIntOperand<i8, "wait_exp", "WaitEXP">;
+def wait_va_vdst : NamedIntOperand<i8, "wait_va_vdst", "WaitVAVDst">;
+def wait_va_vsrc : NamedIntOperand<i8, "wait_vm_vsrc", "WaitVMVSrc">;
class KImmFPOperand<ValueType vt> : ImmOperand<vt> {
let OperandNamespace = "AMDGPU";
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 8310c6b..0f12727 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -30,7 +30,7 @@ include "SMInstructions.td"
include "FLATInstructions.td"
include "BUFInstructions.td"
include "EXPInstructions.td"
-include "LDSDIRInstructions.td"
+include "DSDIRInstructions.td"
include "VINTERPInstructions.td"
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
index 59d6ccf..5e6c349 100644
--- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
+++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
@@ -553,7 +553,9 @@ char SIWholeQuadMode::scanInstructions(MachineFunction &MF,
}
continue;
} else if (Opcode == AMDGPU::LDS_PARAM_LOAD ||
- Opcode == AMDGPU::LDS_DIRECT_LOAD) {
+ Opcode == AMDGPU::DS_PARAM_LOAD ||
+ Opcode == AMDGPU::LDS_DIRECT_LOAD ||
+ Opcode == AMDGPU::DS_DIRECT_LOAD) {
// Mark these STRICTWQM, but only for the instruction, not its operands.
// This avoid unnecessarily marking M0 as requiring WQM.
InstrInfo &II = Instructions[&MI];
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index 3297847..be21cf0 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -977,20 +977,35 @@ def : GCNPat <
}
} // let OtherPredicates = [HasShaderCyclesRegister]
-multiclass SMPrefetchPat<string type, int cache_type> {
+def i32imm_zero : TImmLeaf <i32, [{
+ return Imm == 0;
+}]>;
+
+def i32imm_one : TImmLeaf <i32, [{
+ return Imm == 1;
+}]>;
+
+multiclass SMPrefetchPat<string type, TImmLeaf cache_type> {
def : GCNPat <
- (smrd_prefetch (SMRDImm i64:$sbase, i32:$offset), timm, timm, (i32 cache_type)),
+ (smrd_prefetch (SMRDImm i64:$sbase, i32:$offset), timm, timm, cache_type),
(!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type) $sbase, $offset, (i32 SGPR_NULL), (i8 0))
>;
def : GCNPat <
- (smrd_prefetch (i64 SReg_64:$sbase), timm, timm, (i32 cache_type)),
+ (smrd_prefetch (i64 SReg_64:$sbase), timm, timm, cache_type),
(!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type) $sbase, 0, (i32 SGPR_NULL), (i8 0))
>;
+
+ def : GCNPat <
+ (smrd_prefetch (i32 SReg_32:$sbase), timm, timm, cache_type),
+ (!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type)
+ (i64 (REG_SEQUENCE SReg_64, $sbase, sub0, (i32 (S_MOV_B32 (i32 0))), sub1)),
+ 0, (i32 SGPR_NULL), (i8 0))
+ >;
}
-defm : SMPrefetchPat<"INST", 0>;
-defm : SMPrefetchPat<"DATA", 1>;
+defm : SMPrefetchPat<"INST", i32imm_zero>;
+defm : SMPrefetchPat<"DATA", i32imm_one>;
//===----------------------------------------------------------------------===//
// GFX10.
generated by cgit v1.1 at 2024-07-08 11:25:43 +0000