aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/GlobalISel
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/GlobalISel')
-rw-r--r--llvm/lib/CodeGen/GlobalISel/CallLowering.cpp4
-rw-r--r--llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp87
-rw-r--r--llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp11
-rw-r--r--llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp102
-rw-r--r--llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp67
-rw-r--r--llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp15
-rw-r--r--llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp20
-rw-r--r--llvm/lib/CodeGen/GlobalISel/Legalizer.cpp4
-rw-r--r--llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp53
-rw-r--r--llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp6
-rw-r--r--llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp9
-rw-r--r--llvm/lib/CodeGen/GlobalISel/Utils.cpp27
12 files changed, 316 insertions, 89 deletions
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 7be7468..e2ed45e 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -196,6 +196,10 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
}
+ if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) {
+ Info.DeactivationSymbol = cast<GlobalValue>(Bundle->Inputs[0]);
+ }
+
Info.CB = &CB;
Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees);
Info.CallConv = CallConv;
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index ec4d13f..f0fbe01 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -3463,6 +3463,91 @@ static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits,
isConstTrueVal(TLI, Cst, IsVector, IsFP);
}
+// This pattern aims to match the following shape to avoid extra mov
+// instructions
+// G_BUILD_VECTOR(
+// G_UNMERGE_VALUES(src, 0)
+// G_UNMERGE_VALUES(src, 1)
+// G_IMPLICIT_DEF
+// G_IMPLICIT_DEF
+// )
+// ->
+// G_CONCAT_VECTORS(
+// src,
+// undef
+// )
+bool CombinerHelper::matchCombineBuildUnmerge(MachineInstr &MI,
+ MachineRegisterInfo &MRI,
+ Register &UnmergeSrc) const {
+ auto &BV = cast<GBuildVector>(MI);
+
+ unsigned BuildUseCount = BV.getNumSources();
+ if (BuildUseCount % 2 != 0)
+ return false;
+
+ unsigned NumUnmerge = BuildUseCount / 2;
+
+ auto *Unmerge = getOpcodeDef<GUnmerge>(BV.getSourceReg(0), MRI);
+
+ // Check the first operand is an unmerge and has the correct number of
+ // operands
+ if (!Unmerge || Unmerge->getNumDefs() != NumUnmerge)
+ return false;
+
+ UnmergeSrc = Unmerge->getSourceReg();
+
+ LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
+ LLT UnmergeSrcTy = MRI.getType(UnmergeSrc);
+
+ if (!UnmergeSrcTy.isVector())
+ return false;
+
+ // Ensure we only generate legal instructions post-legalizer
+ if (!IsPreLegalize &&
+ !isLegal({TargetOpcode::G_CONCAT_VECTORS, {DstTy, UnmergeSrcTy}}))
+ return false;
+
+ // Check that all of the operands before the midpoint come from the same
+ // unmerge and are in the same order as they are used in the build_vector
+ for (unsigned I = 0; I < NumUnmerge; ++I) {
+ auto MaybeUnmergeReg = BV.getSourceReg(I);
+ auto *LoopUnmerge = getOpcodeDef<GUnmerge>(MaybeUnmergeReg, MRI);
+
+ if (!LoopUnmerge || LoopUnmerge != Unmerge)
+ return false;
+
+ if (LoopUnmerge->getOperand(I).getReg() != MaybeUnmergeReg)
+ return false;
+ }
+
+ // Check that all of the unmerged values are used
+ if (Unmerge->getNumDefs() != NumUnmerge)
+ return false;
+
+ // Check that all of the operands after the mid point are undefs.
+ for (unsigned I = NumUnmerge; I < BuildUseCount; ++I) {
+ auto *Undef = getDefIgnoringCopies(BV.getSourceReg(I), MRI);
+
+ if (Undef->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
+ return false;
+ }
+
+ return true;
+}
+
+void CombinerHelper::applyCombineBuildUnmerge(MachineInstr &MI,
+ MachineRegisterInfo &MRI,
+ MachineIRBuilder &B,
+ Register &UnmergeSrc) const {
+ assert(UnmergeSrc && "Expected there to be one matching G_UNMERGE_VALUES");
+ B.setInstrAndDebugLoc(MI);
+
+ Register UndefVec = B.buildUndef(MRI.getType(UnmergeSrc)).getReg(0);
+ B.buildConcatVectors(MI.getOperand(0), {UnmergeSrc, UndefVec});
+
+ MI.eraseFromParent();
+}
+
// This combine tries to reduce the number of scalarised G_TRUNC instructions by
// using vector truncates instead
//
@@ -8426,4 +8511,4 @@ bool CombinerHelper::matchSuboCarryOut(const MachineInstr &MI,
}
return false;
-}
+} \ No newline at end of file
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
index c1fb8b6..ecba323 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
@@ -247,6 +247,7 @@ void GISelValueTracking::computeKnownBitsImpl(Register R, KnownBits &Known,
for (unsigned Idx = 1; Idx < MI.getNumOperands(); Idx += 2) {
const MachineOperand &Src = MI.getOperand(Idx);
Register SrcReg = Src.getReg();
+ LLT SrcTy = MRI.getType(SrcReg);
// Look through trivial copies and phis but don't look through trivial
// copies or phis of the form `%1:(s32) = OP %0:gpr32`, known-bits
// analysis is currently unable to determine the bit width of a
@@ -255,9 +256,15 @@ void GISelValueTracking::computeKnownBitsImpl(Register R, KnownBits &Known,
// We can't use NoSubRegister by name as it's defined by each target but
// it's always defined to be 0 by tablegen.
if (SrcReg.isVirtual() && Src.getSubReg() == 0 /*NoSubRegister*/ &&
- MRI.getType(SrcReg).isValid()) {
+ SrcTy.isValid()) {
+ // In case we're forwarding from a vector register to a non-vector
+ // register we need to update the demanded elements to reflect this
+ // before recursing.
+ APInt NowDemandedElts = SrcTy.isFixedVector() && !DstTy.isFixedVector()
+ ? APInt::getAllOnes(SrcTy.getNumElements())
+ : DemandedElts; // Known to be APInt(1, 1)
// For COPYs we don't do anything, don't increase the depth.
- computeKnownBitsImpl(SrcReg, Known2, DemandedElts,
+ computeKnownBitsImpl(SrcReg, Known2, NowDemandedElts,
Depth + (Opcode != TargetOpcode::COPY));
Known2 = Known2.anyextOrTrunc(BitWidth);
Known = Known.intersectWith(Known2);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index d656f10..d87a231 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -111,17 +111,18 @@ INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
false, false)
static void reportTranslationError(MachineFunction &MF,
- const TargetPassConfig &TPC,
OptimizationRemarkEmitter &ORE,
OptimizationRemarkMissed &R) {
MF.getProperties().setFailedISel();
+ bool IsGlobalISelAbortEnabled =
+ MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
// Print the function name explicitly if we don't have a debug location (which
// makes the diagnostic less useful) or if we're going to emit a raw error.
- if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
+ if (!R.getLocation().isValid() || IsGlobalISelAbortEnabled)
R << (" (in function: " + MF.getName() + ")").str();
- if (TPC.isGlobalISelAbortEnabled())
+ if (IsGlobalISelAbortEnabled)
report_fatal_error(Twine(R.getMsg()));
else
ORE.emit(R);
@@ -242,7 +243,7 @@ ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
MF->getFunction().getSubprogram(),
&MF->getFunction().getEntryBlock());
R << "unable to translate constant: " << ore::NV("Type", Val.getType());
- reportTranslationError(*MF, *TPC, *ORE, R);
+ reportTranslationError(*MF, *ORE, R);
return *VRegs;
}
}
@@ -279,7 +280,7 @@ Align IRTranslator::getMemOpAlign(const Instruction &I) {
OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
R << "unable to translate memop: " << ore::NV("Opcode", &I);
- reportTranslationError(*MF, *TPC, *ORE, R);
+ reportTranslationError(*MF, *ORE, R);
return Align(1);
}
@@ -1412,14 +1413,14 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
for (unsigned i = 0; i < Regs.size(); ++i) {
Register Addr;
- MIRBuilder.materializeObjectPtrOffset(Addr, Base, OffsetTy, Offsets[i] / 8);
+ MIRBuilder.materializeObjectPtrOffset(Addr, Base, OffsetTy, Offsets[i]);
- MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
+ MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i]);
Align BaseAlign = getMemOpAlign(LI);
- auto MMO = MF->getMachineMemOperand(
- Ptr, Flags, MRI->getType(Regs[i]),
- commonAlignment(BaseAlign, Offsets[i] / 8), AAInfo, Ranges,
- LI.getSyncScopeID(), LI.getOrdering());
+ auto MMO =
+ MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Regs[i]),
+ commonAlignment(BaseAlign, Offsets[i]), AAInfo,
+ Ranges, LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
}
@@ -1451,14 +1452,14 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
for (unsigned i = 0; i < Vals.size(); ++i) {
Register Addr;
- MIRBuilder.materializeObjectPtrOffset(Addr, Base, OffsetTy, Offsets[i] / 8);
+ MIRBuilder.materializeObjectPtrOffset(Addr, Base, OffsetTy, Offsets[i]);
- MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
+ MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i]);
Align BaseAlign = getMemOpAlign(SI);
- auto MMO = MF->getMachineMemOperand(
- Ptr, Flags, MRI->getType(Vals[i]),
- commonAlignment(BaseAlign, Offsets[i] / 8), SI.getAAMetadata(), nullptr,
- SI.getSyncScopeID(), SI.getOrdering());
+ auto MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Vals[i]),
+ commonAlignment(BaseAlign, Offsets[i]),
+ SI.getAAMetadata(), nullptr,
+ SI.getSyncScopeID(), SI.getOrdering());
MIRBuilder.buildStore(Vals[i], Addr, *MMO);
}
return true;
@@ -1483,8 +1484,8 @@ static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
llvm::append_range(Indices, drop_begin(U.operands()));
}
- return 8 * static_cast<uint64_t>(
- DL.getIndexedOffsetInType(Src->getType(), Indices));
+ return static_cast<uint64_t>(
+ DL.getIndexedOffsetInType(Src->getType(), Indices));
}
bool IRTranslator::translateExtractValue(const User &U,
@@ -2835,7 +2836,7 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
IsTgtMemIntrinsic ? &Info : nullptr);
}
-/// Translate a call to an intrinsic.
+/// Translate a call or callbr to an intrinsic.
/// Depending on whether TLI->getTgtMemIntrinsic() is true, TgtMemIntrinsicInfo
/// is a pointer to the correspondingly populated IntrinsicInfo object.
/// Otherwise, this pointer is null.
@@ -2917,6 +2918,9 @@ bool IRTranslator::translateIntrinsic(
}
}
+ if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol))
+ MIB->setDeactivationSymbol(*MF, Bundle->Inputs[0].get());
+
return true;
}
@@ -3063,10 +3067,52 @@ bool IRTranslator::translateInvoke(const User &U,
return true;
}
+/// The intrinsics currently supported by callbr are implicit control flow
+/// intrinsics such as amdgcn.kill.
bool IRTranslator::translateCallBr(const User &U,
MachineIRBuilder &MIRBuilder) {
- // FIXME: Implement this.
- return false;
+ if (containsBF16Type(U))
+ return false; // see translateCall
+
+ const CallBrInst &I = cast<CallBrInst>(U);
+ MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB();
+
+ Intrinsic::ID IID = I.getIntrinsicID();
+ if (I.isInlineAsm()) {
+ // FIXME: inline asm is not yet supported for callbr in GlobalISel. As soon
+ // as we add support, we need to handle the indirect asm targets, see
+ // SelectionDAGBuilder::visitCallBr().
+ return false;
+ }
+ if (!translateIntrinsic(I, IID, MIRBuilder))
+ return false;
+
+ // Retrieve successors.
+ SmallPtrSet<BasicBlock *, 8> Dests = {I.getDefaultDest()};
+ MachineBasicBlock *Return = &getMBB(*I.getDefaultDest());
+
+ // Update successor info.
+ addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
+
+ // Add indirect targets as successors. For intrinsic callbr, these represent
+ // implicit control flow (e.g., the "kill" path for amdgcn.kill). We mark them
+ // with setIsInlineAsmBrIndirectTarget so the machine verifier accepts them as
+ // valid successors, even though they're not from inline asm.
+ for (BasicBlock *Dest : I.getIndirectDests()) {
+ MachineBasicBlock &Target = getMBB(*Dest);
+ Target.setIsInlineAsmBrIndirectTarget();
+ Target.setLabelMustBeEmitted();
+ // Don't add duplicate machine successors.
+ if (Dests.insert(Dest).second)
+ addSuccessorWithProb(CallBrMBB, &Target, BranchProbability::getZero());
+ }
+
+ CallBrMBB->normalizeSuccProbs();
+
+ // Drop into default successor.
+ MIRBuilder.buildBr(*Return);
+
+ return true;
}
bool IRTranslator::translateLandingPad(const User &U,
@@ -3482,7 +3528,7 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
bool IRTranslator::translateAtomicRMW(const User &U,
MachineIRBuilder &MIRBuilder) {
- if (!MF->getTarget().getTargetTriple().isSPIRV() && containsBF16Type(U))
+ if (containsBF16Type(U) && !targetSupportsBF16Type(MF))
return false;
const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
@@ -4144,7 +4190,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
F.getSubprogram(), &F.getEntryBlock());
R << "unable to translate in big endian mode";
- reportTranslationError(*MF, *TPC, *ORE, R);
+ reportTranslationError(*MF, *ORE, R);
return false;
}
@@ -4188,7 +4234,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
F.getSubprogram(), &F.getEntryBlock());
R << "unable to lower function: "
<< ore::NV("Prototype", F.getFunctionType());
- reportTranslationError(*MF, *TPC, *ORE, R);
+ reportTranslationError(*MF, *ORE, R);
return false;
}
@@ -4211,7 +4257,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
F.getSubprogram(), &F.getEntryBlock());
R << "unable to lower arguments: "
<< ore::NV("Prototype", F.getFunctionType());
- reportTranslationError(*MF, *TPC, *ORE, R);
+ reportTranslationError(*MF, *ORE, R);
return false;
}
@@ -4262,7 +4308,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
R << ": '" << InstStrStorage << "'";
}
- reportTranslationError(*MF, *TPC, *ORE, R);
+ reportTranslationError(*MF, *ORE, R);
return false;
}
@@ -4270,7 +4316,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
BB->getTerminator()->getDebugLoc(), BB);
R << "unable to translate basic block";
- reportTranslationError(*MF, *TPC, *ORE, R);
+ reportTranslationError(*MF, *ORE, R);
return false;
}
}
diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
index b4e64d7..f695a2d 100644
--- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
@@ -13,6 +13,7 @@
#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
@@ -454,26 +455,52 @@ bool InlineAsmLowering::lowerInlineAsm(
}
if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
-
- if (!OpInfo.isIndirect) {
- LLVM_DEBUG(dbgs()
- << "Cannot indirectify memory input operands yet\n");
- return false;
- }
-
- assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!");
-
const InlineAsm::ConstraintCode ConstraintID =
TLI->getInlineAsmMemConstraint(OpInfo.ConstraintCode);
InlineAsm::Flag OpFlags(InlineAsm::Kind::Mem, 1);
OpFlags.setMemConstraint(ConstraintID);
Inst.addImm(OpFlags);
+
+ if (OpInfo.isIndirect) {
+ // already indirect
+ ArrayRef<Register> SourceRegs =
+ GetOrCreateVRegs(*OpInfo.CallOperandVal);
+ if (SourceRegs.size() != 1) {
+ LLVM_DEBUG(dbgs() << "Expected the memory input to fit into a "
+ "single virtual register "
+ "for constraint '"
+ << OpInfo.ConstraintCode << "'\n");
+ return false;
+ }
+ Inst.addReg(SourceRegs[0]);
+ break;
+ }
+
+ // Needs to be made indirect. Store the value on the stack and use
+ // a pointer to it.
+ Value *OpVal = OpInfo.CallOperandVal;
+ TypeSize Bytes = DL.getTypeStoreSize(OpVal->getType());
+ Align Alignment = DL.getPrefTypeAlign(OpVal->getType());
+ int FrameIdx =
+ MF.getFrameInfo().CreateStackObject(Bytes, Alignment, false);
+
+ unsigned AddrSpace = DL.getAllocaAddrSpace();
+ LLT FramePtrTy =
+ LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace));
+ auto Ptr = MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx).getReg(0);
ArrayRef<Register> SourceRegs =
GetOrCreateVRegs(*OpInfo.CallOperandVal);
- assert(
- SourceRegs.size() == 1 &&
- "Expected the memory input to fit into a single virtual register");
- Inst.addReg(SourceRegs[0]);
+ if (SourceRegs.size() != 1) {
+ LLVM_DEBUG(dbgs() << "Expected the memory input to fit into a single "
+ "virtual register "
+ "for constraint '"
+ << OpInfo.ConstraintCode << "'\n");
+ return false;
+ }
+ MIRBuilder.buildStore(SourceRegs[0], Ptr,
+ MachinePointerInfo::getFixedStack(MF, FrameIdx),
+ Alignment);
+ Inst.addReg(Ptr);
break;
}
@@ -538,13 +565,6 @@ bool InlineAsmLowering::lowerInlineAsm(
}
}
- // Add rounding control registers as implicit def for inline asm.
- if (MF.getFunction().hasFnAttribute(Attribute::StrictFP)) {
- ArrayRef<MCPhysReg> RCRegs = TLI->getRoundingControlRegisters();
- for (MCPhysReg Reg : RCRegs)
- Inst.addReg(Reg, RegState::ImplicitDefine);
- }
-
if (auto Bundle = Call.getOperandBundle(LLVMContext::OB_convergencectrl)) {
auto *Token = Bundle->Inputs[0].get();
ArrayRef<Register> SourceRegs = GetOrCreateVRegs(*Token);
@@ -556,6 +576,13 @@ bool InlineAsmLowering::lowerInlineAsm(
if (const MDNode *SrcLoc = Call.getMetadata("srcloc"))
Inst.addMetadata(SrcLoc);
+ // Add rounding control registers as implicit def for inline asm.
+ if (MF.getFunction().hasFnAttribute(Attribute::StrictFP)) {
+ ArrayRef<MCPhysReg> RCRegs = TLI->getRoundingControlRegisters();
+ for (MCPhysReg Reg : RCRegs)
+ Inst.addReg(Reg, RegState::ImplicitDefine);
+ }
+
// All inputs are handled, insert the instruction now
MIRBuilder.insertInstr(Inst);
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index 2dd22c8a..1d281ab 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -137,7 +137,6 @@ bool InstructionSelect::runOnMachineFunction(MachineFunction &MF) {
return false;
ISel = MF.getSubtarget().getInstructionSelector();
- ISel->TPC = &getAnalysis<TargetPassConfig>();
// FIXME: Properly override OptLevel in TargetMachine. See OptLevelChanger
CodeGenOptLevel OldOptLevel = OptLevel;
@@ -159,7 +158,6 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
LLVM_DEBUG(dbgs() << "Selecting function: " << MF.getName() << '\n');
assert(ISel && "Cannot work without InstructionSelector");
- const TargetPassConfig &TPC = *ISel->TPC;
CodeGenCoverage CoverageInfo;
ISel->setupMF(MF, VT, &CoverageInfo, PSI, BFI);
@@ -177,8 +175,8 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
// property check already is.
if (!DisableGISelLegalityCheck)
if (const MachineInstr *MI = machineFunctionIsIllegal(MF)) {
- reportGISelFailure(MF, TPC, MORE, "gisel-select",
- "instruction is not legal", *MI);
+ reportGISelFailure(MF, MORE, "gisel-select", "instruction is not legal",
+ *MI);
return false;
}
// FIXME: We could introduce new blocks and will need to fix the outer loop.
@@ -215,8 +213,7 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
if (!selectInstr(MI)) {
LLVM_DEBUG(dbgs() << "Selection failed!\n";
MIIMaintainer.reportFullyCreatedInstrs());
- reportGISelFailure(MF, TPC, MORE, "gisel-select", "cannot select",
- MI);
+ reportGISelFailure(MF, MORE, "gisel-select", "cannot select", MI);
return false;
}
LLVM_DEBUG(MIIMaintainer.reportFullyCreatedInstrs());
@@ -279,7 +276,7 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
const TargetRegisterClass *RC = MRI.getRegClassOrNull(VReg);
if (!RC) {
- reportGISelFailure(MF, TPC, MORE, "gisel-select",
+ reportGISelFailure(MF, MORE, "gisel-select",
"VReg has no regclass after selection", *MI);
return false;
}
@@ -288,7 +285,7 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
if (Ty.isValid() &&
TypeSize::isKnownGT(Ty.getSizeInBits(), TRI.getRegSizeInBits(*RC))) {
reportGISelFailure(
- MF, TPC, MORE, "gisel-select",
+ MF, MORE, "gisel-select",
"VReg's low-level type and register class have different sizes", *MI);
return false;
}
@@ -299,7 +296,7 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
MF.getFunction().getSubprogram(),
/*MBB=*/nullptr);
R << "inserting blocks is not supported yet";
- reportGISelFailure(MF, TPC, MORE, R);
+ reportGISelFailure(MF, MORE, R);
return false;
}
#endif
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
index 30c2d08..5e7cd5f 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
@@ -155,6 +155,26 @@ LegalityPredicate LegalityPredicates::scalarOrEltNarrowerThan(unsigned TypeIdx,
};
}
+LegalityPredicate
+LegalityPredicates::vectorElementCountIsGreaterThan(unsigned TypeIdx,
+ unsigned Size) {
+
+ return [=](const LegalityQuery &Query) {
+ const LLT QueryTy = Query.Types[TypeIdx];
+ return QueryTy.isFixedVector() && QueryTy.getNumElements() > Size;
+ };
+}
+
+LegalityPredicate
+LegalityPredicates::vectorElementCountIsLessThanOrEqualTo(unsigned TypeIdx,
+ unsigned Size) {
+
+ return [=](const LegalityQuery &Query) {
+ const LLT QueryTy = Query.Types[TypeIdx];
+ return QueryTy.isFixedVector() && QueryTy.getNumElements() <= Size;
+ };
+}
+
LegalityPredicate LegalityPredicates::scalarOrEltWiderThan(unsigned TypeIdx,
unsigned Size) {
return [=](const LegalityQuery &Query) {
diff --git a/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp b/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp
index aef16b5..0f0656a 100644
--- a/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp
@@ -348,7 +348,7 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
*MIRBuilder, VT);
if (Result.FailedOn) {
- reportGISelFailure(MF, TPC, MORE, "gisel-legalize",
+ reportGISelFailure(MF, MORE, "gisel-legalize",
"unable to legalize instruction", *Result.FailedOn);
return false;
}
@@ -360,7 +360,7 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
R << "lost "
<< ore::NV("NumLostDebugLocs", LocObserver.getNumLostDebugLocs())
<< " debug locations during pass";
- reportGISelWarning(MF, TPC, MORE, R);
+ reportGISelWarning(MF, MORE, R);
// Example remark:
// --- !Missed
// Pass: gisel-legalize
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index d02f097..251ea4b 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1930,7 +1930,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
Register CmpOut;
CmpInst::Predicate PartPred;
- if (I == E - 1 && LHSLeftoverRegs.empty()) {
+ if (I == E - 1) {
PartPred = Pred;
CmpOut = Dst;
} else {
@@ -3065,6 +3065,14 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changedInstr(MI);
return Legalized;
+ case TargetOpcode::G_FPEXT:
+ if (TypeIdx != 1)
+ return UnableToLegalize;
+
+ Observer.changingInstr(MI);
+ widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
+ Observer.changedInstr(MI);
+ return Legalized;
case TargetOpcode::G_FPTOSI:
case TargetOpcode::G_FPTOUI:
case TargetOpcode::G_INTRINSIC_LRINT:
@@ -3431,6 +3439,18 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changedInstr(MI);
return Legalized;
}
+ case TargetOpcode::G_LROUND:
+ case TargetOpcode::G_LLROUND:
+ Observer.changingInstr(MI);
+
+ if (TypeIdx == 0)
+ widenScalarDst(MI, WideTy);
+ else
+ widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
+
+ Observer.changedInstr(MI);
+ return Legalized;
+
case TargetOpcode::G_INTTOPTR:
if (TypeIdx != 1)
return UnableToLegalize;
@@ -6664,13 +6684,24 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
case TargetOpcode::G_FMAXIMUMNUM:
case TargetOpcode::G_STRICT_FADD:
case TargetOpcode::G_STRICT_FSUB:
- case TargetOpcode::G_STRICT_FMUL:
+ case TargetOpcode::G_STRICT_FMUL: {
+ Observer.changingInstr(MI);
+ moreElementsVectorSrc(MI, MoreTy, 1);
+ moreElementsVectorSrc(MI, MoreTy, 2);
+ moreElementsVectorDst(MI, MoreTy, 0);
+ Observer.changedInstr(MI);
+ return Legalized;
+ }
case TargetOpcode::G_SHL:
case TargetOpcode::G_ASHR:
case TargetOpcode::G_LSHR: {
Observer.changingInstr(MI);
moreElementsVectorSrc(MI, MoreTy, 1);
- moreElementsVectorSrc(MI, MoreTy, 2);
+ // The shift operand may have a different scalar type from the source and
+ // destination operands.
+ LLT ShiftMoreTy = MoreTy.changeElementType(
+ MRI.getType(MI.getOperand(2).getReg()).getElementType());
+ moreElementsVectorSrc(MI, ShiftMoreTy, 2);
moreElementsVectorDst(MI, MoreTy, 0);
Observer.changedInstr(MI);
return Legalized;
@@ -6786,12 +6817,10 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
LLT DstExtTy;
if (TypeIdx == 0) {
DstExtTy = MoreTy;
- SrcExtTy = LLT::fixed_vector(
- MoreTy.getNumElements(),
+ SrcExtTy = MoreTy.changeElementType(
MRI.getType(MI.getOperand(1).getReg()).getElementType());
} else {
- DstExtTy = LLT::fixed_vector(
- MoreTy.getNumElements(),
+ DstExtTy = MoreTy.changeElementType(
MRI.getType(MI.getOperand(0).getReg()).getElementType());
SrcExtTy = MoreTy;
}
@@ -7589,7 +7618,7 @@ LegalizerHelper::lowerBitCount(MachineInstr &MI) {
}
case TargetOpcode::G_CTLZ: {
auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
- unsigned Len = SrcTy.getSizeInBits();
+ unsigned Len = SrcTy.getScalarSizeInBits();
if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
// If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
@@ -7637,7 +7666,7 @@ LegalizerHelper::lowerBitCount(MachineInstr &MI) {
case TargetOpcode::G_CTTZ: {
auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
- unsigned Len = SrcTy.getSizeInBits();
+ unsigned Len = SrcTy.getScalarSizeInBits();
if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
// If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
// zero.
@@ -7675,9 +7704,13 @@ LegalizerHelper::lowerBitCount(MachineInstr &MI) {
case TargetOpcode::G_CTPOP: {
Register SrcReg = MI.getOperand(1).getReg();
LLT Ty = MRI.getType(SrcReg);
- unsigned Size = Ty.getSizeInBits();
+ unsigned Size = Ty.getScalarSizeInBits();
MachineIRBuilder &B = MIRBuilder;
+ // Bail out on irregular type lengths.
+ if (Size > 128 || Size % 8 != 0)
+ return UnableToLegalize;
+
// Count set bits in blocks of 2 bits. Default approach would be
// B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 }
// We use following formula instead:
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 637acd6..3906b31 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -38,8 +38,10 @@ void MachineIRBuilder::setMF(MachineFunction &MF) {
//------------------------------------------------------------------------------
MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) {
- return BuildMI(getMF(), {getDL(), getPCSections(), getMMRAMetadata()},
- getTII().get(Opcode));
+ return BuildMI(
+ getMF(),
+ {getDL(), getPCSections(), getMMRAMetadata(), getDeactivationSymbol()},
+ getTII().get(Opcode));
}
MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) {
diff --git a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
index bcb4f1c..5db631b 100644
--- a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
@@ -39,6 +39,7 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
@@ -83,7 +84,6 @@ void RegBankSelect::init(MachineFunction &MF) {
assert(RBI && "Cannot work without RegisterBankInfo");
MRI = &MF.getRegInfo();
TRI = MF.getSubtarget().getRegisterInfo();
- TPC = &getAnalysis<TargetPassConfig>();
if (OptMode != Mode::Fast) {
MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
@@ -308,7 +308,8 @@ const RegisterBankInfo::InstructionMapping &RegBankSelect::findBestMapping(
RepairPts.emplace_back(std::move(RepairPt));
}
}
- if (!BestMapping && !TPC->isGlobalISelAbortEnabled()) {
+ if (!BestMapping && MI.getMF()->getTarget().Options.GlobalISelAbort !=
+ GlobalISelAbortMode::Enable) {
// If none of the mapping worked that means they are all impossible.
// Thus, pick the first one and set an impossible repairing point.
// It will trigger the failed isel mode.
@@ -708,7 +709,7 @@ bool RegBankSelect::assignRegisterBanks(MachineFunction &MF) {
continue;
if (!assignInstr(MI)) {
- reportGISelFailure(MF, *TPC, *MORE, "gisel-regbankselect",
+ reportGISelFailure(MF, *MORE, "gisel-regbankselect",
"unable to map instruction", MI);
return false;
}
@@ -722,7 +723,7 @@ bool RegBankSelect::checkFunctionIsLegal(MachineFunction &MF) const {
#ifndef NDEBUG
if (!DisableGISelLegalityCheck) {
if (const MachineInstr *MI = machineFunctionIsIllegal(MF)) {
- reportGISelFailure(MF, *TPC, *MORE, "gisel-regbankselect",
+ reportGISelFailure(MF, *MORE, "gisel-regbankselect",
"instruction is not legal", *MI);
return false;
}
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 5fab6ec..15e81f5 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -114,7 +114,7 @@ Register llvm::constrainOperandRegClass(
// Assume physical registers are properly constrained.
assert(Reg.isVirtual() && "PhysReg not implemented");
- const TargetRegisterClass *OpRC = TII.getRegClass(II, OpIdx, &TRI);
+ const TargetRegisterClass *OpRC = TII.getRegClass(II, OpIdx);
// Some of the target independent instructions, like COPY, may not impose any
// register class constraints on some of their operands: If it's a use, we can
// skip constraining as the instruction defining the register would constrain
@@ -234,11 +234,11 @@ bool llvm::isTriviallyDead(const MachineInstr &MI,
static void reportGISelDiagnostic(DiagnosticSeverity Severity,
MachineFunction &MF,
- const TargetPassConfig &TPC,
MachineOptimizationRemarkEmitter &MORE,
MachineOptimizationRemarkMissed &R) {
- bool IsFatal = Severity == DS_Error &&
- TPC.isGlobalISelAbortEnabled();
+ bool IsGlobalISelAbortEnabled =
+ MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
+ bool IsFatal = Severity == DS_Error && IsGlobalISelAbortEnabled;
// Print the function name explicitly if we don't have a debug location (which
// makes the diagnostic less useful) or if we're going to emit a raw error.
if (!R.getLocation().isValid() || IsFatal)
@@ -250,20 +250,20 @@ static void reportGISelDiagnostic(DiagnosticSeverity Severity,
MORE.emit(R);
}
-void llvm::reportGISelWarning(MachineFunction &MF, const TargetPassConfig &TPC,
+void llvm::reportGISelWarning(MachineFunction &MF,
MachineOptimizationRemarkEmitter &MORE,
MachineOptimizationRemarkMissed &R) {
- reportGISelDiagnostic(DS_Warning, MF, TPC, MORE, R);
+ reportGISelDiagnostic(DS_Warning, MF, MORE, R);
}
-void llvm::reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC,
+void llvm::reportGISelFailure(MachineFunction &MF,
MachineOptimizationRemarkEmitter &MORE,
MachineOptimizationRemarkMissed &R) {
MF.getProperties().setFailedISel();
- reportGISelDiagnostic(DS_Error, MF, TPC, MORE, R);
+ reportGISelDiagnostic(DS_Error, MF, MORE, R);
}
-void llvm::reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC,
+void llvm::reportGISelFailure(MachineFunction &MF,
MachineOptimizationRemarkEmitter &MORE,
const char *PassName, StringRef Msg,
const MachineInstr &MI) {
@@ -271,9 +271,10 @@ void llvm::reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC,
MI.getDebugLoc(), MI.getParent());
R << Msg;
// Printing MI is expensive; only do it if expensive remarks are enabled.
- if (TPC.isGlobalISelAbortEnabled() || MORE.allowExtraAnalysis(PassName))
+ if (MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable ||
+ MORE.allowExtraAnalysis(PassName))
R << ": " << ore::MNV("Inst", MI);
- reportGISelFailure(MF, TPC, MORE, R);
+ reportGISelFailure(MF, MORE, R);
}
unsigned llvm::getInverseGMinMaxOpcode(unsigned MinMaxOpc) {
@@ -768,8 +769,12 @@ llvm::ConstantFoldFPBinOp(unsigned Opcode, const Register Op1,
C1.copySign(C2);
return C1;
case TargetOpcode::G_FMINNUM:
+ if (C1.isSignaling() || C2.isSignaling())
+ return std::nullopt;
return minnum(C1, C2);
case TargetOpcode::G_FMAXNUM:
+ if (C1.isSignaling() || C2.isSignaling())
+ return std::nullopt;
return maxnum(C1, C2);
case TargetOpcode::G_FMINIMUM:
return minimum(C1, C2);
generated by cgit v1.1 at 2026-01-05 12:45:10 +0000