diff options
Diffstat (limited to 'llvm/lib/CodeGen')
24 files changed, 404 insertions, 320 deletions
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 76a1d8c..f1d3e96 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -809,7 +809,7 @@ void AsmPrinter::emitGlobalVariable(const GlobalVariable *GV) { // If we have a bss global going to a section that supports the // zerofill directive, do so here. - if (GVKind.isBSS() && MAI->isMachO() && TheSection->isVirtualSection()) { + if (GVKind.isBSS() && MAI->isMachO() && TheSection->isBssSection()) { if (Size == 0) Size = 1; // zerofill of 0 bytes is undefined. emitLinkage(GV, GVSym); @@ -1868,6 +1868,7 @@ void AsmPrinter::emitFunctionBody() { OutStreamer->emitLabel(MI.getOperand(0).getMCSymbol()); break; case TargetOpcode::EH_LABEL: + OutStreamer->AddComment("EH_LABEL"); OutStreamer->emitLabel(MI.getOperand(0).getMCSymbol()); // For AsynchEH, insert a Nop if followed by a trap inst // Or the exception won't be caught. diff --git a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp index 618deef..4bf3bdf 100644 --- a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp @@ -18,6 +18,11 @@ #include "llvm/MC/MCPseudoProbe.h" #include "llvm/MC/MCStreamer.h" +#ifndef NDEBUG +#include "llvm/IR/Module.h" +#include "llvm/Support/WithColor.h" +#endif + using namespace llvm; void PseudoProbeHandler::emitPseudoProbe(uint64_t Guid, uint64_t Index, @@ -35,6 +40,9 @@ void PseudoProbeHandler::emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t &CallerGuid = NameGuidMap[Name]; if (!CallerGuid) CallerGuid = Function::getGUIDAssumingExternalLinkage(Name); +#ifndef NDEBUG + verifyGuidExistenceInDesc(CallerGuid, Name); +#endif uint64_t CallerProbeId = PseudoProbeDwarfDiscriminator::extractProbeIndex( InlinedAt->getDiscriminator()); ReversedInlineStack.emplace_back(CallerGuid, CallerProbeId); @@ -51,4 +59,28 @@ void PseudoProbeHandler::emitPseudoProbe(uint64_t Guid, uint64_t Index, SmallVector<InlineSite, 8> InlineStack(llvm::reverse(ReversedInlineStack)); Asm->OutStreamer->emitPseudoProbe(Guid, Index, Type, Attr, Discriminator, InlineStack, Asm->CurrentFnSym); +#ifndef NDEBUG + verifyGuidExistenceInDesc( + Guid, DebugLoc ? DebugLoc->getSubprogramLinkageName() : ""); +#endif +} + +#ifndef NDEBUG +void PseudoProbeHandler::verifyGuidExistenceInDesc(uint64_t Guid, + StringRef FuncName) { + NamedMDNode *Desc = Asm->MF->getFunction().getParent()->getNamedMetadata( + PseudoProbeDescMetadataName); + assert(Desc && "pseudo probe does not exist"); + + // Keep DescGuidSet up to date. + for (size_t I = DescGuidSet.size(), E = Desc->getNumOperands(); I != E; ++I) { + const auto *MD = cast<MDNode>(Desc->getOperand(I)); + auto *ID = mdconst::extract<ConstantInt>(MD->getOperand(0)); + DescGuidSet.insert(ID->getZExtValue()); + } + + if (!DescGuidSet.contains(Guid)) + WithColor::warning() << "Guid:" << Guid << " Name:" << FuncName + << " does not exist in pseudo probe desc\n"; } +#endif diff --git a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.h b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.h index f11b552..e950b23 100644 --- a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.h +++ b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.h @@ -15,6 +15,10 @@ #include "llvm/ADT/DenseMap.h" +#ifndef NDEBUG +#include "llvm/ADT/DenseSet.h" +#endif + namespace llvm { class AsmPrinter; @@ -26,6 +30,13 @@ class PseudoProbeHandler { // Name to GUID map, used as caching/memoization for speed. DenseMap<StringRef, uint64_t> NameGuidMap; +#ifndef NDEBUG + // All GUID in llvm.pseudo_probe_desc. + DenseSet<uint64_t> DescGuidSet; + + void verifyGuidExistenceInDesc(uint64_t Guid, StringRef FuncName); +#endif + public: PseudoProbeHandler(AsmPrinter *A) : Asm(A) {}; diff --git a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp index dccd71f..13fd270 100644 --- a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp @@ -323,12 +323,6 @@ const MCExpr *WinException::getLabel(const MCSymbol *Label) { Asm->OutContext); } -const MCExpr *WinException::getLabelPlusOne(const MCSymbol *Label) { - return MCBinaryExpr::createAdd(getLabel(Label), - MCConstantExpr::create(1, Asm->OutContext), - Asm->OutContext); -} - const MCExpr *WinException::getOffset(const MCSymbol *OffsetOf, const MCSymbol *OffsetFrom) { return MCBinaryExpr::createSub( @@ -655,7 +649,7 @@ void WinException::emitSEHActionsForRange(const WinEHFuncInfo &FuncInfo, AddComment("LabelStart"); OS.emitValue(getLabel(BeginLabel), 4); AddComment("LabelEnd"); - OS.emitValue(getLabelPlusOne(EndLabel), 4); + OS.emitValue(getLabel(EndLabel), 4); AddComment(UME.IsFinally ? "FinallyFunclet" : UME.Filter ? "FilterFunction" : "CatchAll"); OS.emitValue(FilterOrFinally, 4); @@ -950,13 +944,7 @@ void WinException::computeIP2StateTable( if (!ChangeLabel) ChangeLabel = StateChange.PreviousEndLabel; // Emit an entry indicating that PCs after 'Label' have this EH state. - // NOTE: On ARM architectures, the StateFromIp automatically takes into - // account that the return address is after the call instruction (whose EH - // state we should be using), but on other platforms we need to +1 to the - // label so that we are using the correct EH state. - const MCExpr *LabelExpression = (isAArch64 || isThumb) - ? getLabel(ChangeLabel) - : getLabelPlusOne(ChangeLabel); + const MCExpr *LabelExpression = getLabel(ChangeLabel); IPToStateTable.push_back( std::make_pair(LabelExpression, StateChange.NewState)); // FIXME: assert that NewState is between CatchLow and CatchHigh. diff --git a/llvm/lib/CodeGen/AsmPrinter/WinException.h b/llvm/lib/CodeGen/AsmPrinter/WinException.h index 638589a..47dd30c 100644 --- a/llvm/lib/CodeGen/AsmPrinter/WinException.h +++ b/llvm/lib/CodeGen/AsmPrinter/WinException.h @@ -80,7 +80,6 @@ class LLVM_LIBRARY_VISIBILITY WinException : public EHStreamer { const MCExpr *create32bitRef(const MCSymbol *Value); const MCExpr *create32bitRef(const GlobalValue *GV); const MCExpr *getLabel(const MCSymbol *Label); - const MCExpr *getLabelPlusOne(const MCSymbol *Label); const MCExpr *getOffset(const MCSymbol *OffsetOf, const MCSymbol *OffsetFrom); const MCExpr *getOffsetPlusOne(const MCSymbol *OffsetOf, const MCSymbol *OffsetFrom); diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index dc81843..c21058c 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -3571,9 +3571,7 @@ class TypePromotionTransaction { } // Record the debug uses separately. They are not in the instruction's // use list, but they are replaced by RAUW. - SmallVector<DbgValueInst *> DbgValues; - findDbgValues(DbgValues, Inst, &DbgVariableRecords); - assert(DbgValues.empty()); + findDbgValues(Inst, DbgVariableRecords); // Now, we can replace the uses. Inst->replaceAllUsesWith(New); diff --git a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp index 1286af8..974fc40 100644 --- a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp +++ b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp @@ -1884,6 +1884,14 @@ unsigned GISelValueTracking::computeNumSignBits(Register R, } break; } + case TargetOpcode::G_ASHR: { + Register Src1 = MI.getOperand(1).getReg(); + Register Src2 = MI.getOperand(2).getReg(); + FirstAnswer = computeNumSignBits(Src1, DemandedElts, Depth + 1); + if (auto C = getValidMinimumShiftAmount(Src2, DemandedElts, Depth + 1)) + FirstAnswer = std::min<uint64_t>(FirstAnswer + *C, TyBits); + break; + } case TargetOpcode::G_TRUNC: { Register Src = MI.getOperand(1).getReg(); LLT SrcTy = MRI.getType(Src); @@ -2053,6 +2061,64 @@ unsigned GISelValueTracking::computeNumSignBits(Register R, unsigned Depth) { return computeNumSignBits(R, DemandedElts, Depth); } +std::optional<ConstantRange> GISelValueTracking::getValidShiftAmountRange( + Register R, const APInt &DemandedElts, unsigned Depth) { + // Shifting more than the bitwidth is not valid. + MachineInstr &MI = *MRI.getVRegDef(R); + unsigned Opcode = MI.getOpcode(); + + LLT Ty = MRI.getType(R); + unsigned BitWidth = Ty.getScalarSizeInBits(); + + if (Opcode == TargetOpcode::G_CONSTANT) { + const APInt &ShAmt = MI.getOperand(1).getCImm()->getValue(); + if (ShAmt.uge(BitWidth)) + return std::nullopt; + return ConstantRange(ShAmt); + } + + if (Opcode == TargetOpcode::G_BUILD_VECTOR) { + const APInt *MinAmt = nullptr, *MaxAmt = nullptr; + for (unsigned I = 0, E = MI.getNumOperands() - 1; I != E; ++I) { + if (!DemandedElts[I]) + continue; + MachineInstr *Op = MRI.getVRegDef(MI.getOperand(I + 1).getReg()); + if (Op->getOpcode() != TargetOpcode::G_CONSTANT) { + MinAmt = MaxAmt = nullptr; + break; + } + + const APInt &ShAmt = Op->getOperand(1).getCImm()->getValue(); + if (ShAmt.uge(BitWidth)) + return std::nullopt; + if (!MinAmt || MinAmt->ugt(ShAmt)) + MinAmt = &ShAmt; + if (!MaxAmt || MaxAmt->ult(ShAmt)) + MaxAmt = &ShAmt; + } + assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) && + "Failed to find matching min/max shift amounts"); + if (MinAmt && MaxAmt) + return ConstantRange(*MinAmt, *MaxAmt + 1); + } + + // Use computeKnownBits to find a hidden constant/knownbits (usually type + // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc. + KnownBits KnownAmt = getKnownBits(R, DemandedElts, Depth); + if (KnownAmt.getMaxValue().ult(BitWidth)) + return ConstantRange::fromKnownBits(KnownAmt, /*IsSigned=*/false); + + return std::nullopt; +} + +std::optional<uint64_t> GISelValueTracking::getValidMinimumShiftAmount( + Register R, const APInt &DemandedElts, unsigned Depth) { + if (std::optional<ConstantRange> AmtRange = + getValidShiftAmountRange(R, DemandedElts, Depth)) + return AmtRange->getUnsignedMin().getZExtValue(); + return std::nullopt; +} + void GISelValueTrackingAnalysisLegacy::getAnalysisUsage( AnalysisUsage &AU) const { AU.setPreservesAll(); diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index d7280ea..dc5dfab 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -2189,23 +2189,11 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START : TargetOpcode::LIFETIME_END; - // Get the underlying objects for the location passed on the lifetime - // marker. - SmallVector<const Value *, 4> Allocas; - getUnderlyingObjects(CI.getArgOperand(1), Allocas); - - // Iterate over each underlying object, creating lifetime markers for each - // static alloca. Quit if we find a non-static alloca. - for (const Value *V : Allocas) { - const AllocaInst *AI = dyn_cast<AllocaInst>(V); - if (!AI) - continue; - - if (!AI->isStaticAlloca()) - return true; + const AllocaInst *AI = cast<AllocaInst>(CI.getArgOperand(1)); + if (!AI->isStaticAlloca()) + return true; - MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI)); - } + MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI)); return true; } case Intrinsic::fake_use: { diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 11b3ac8..ed7b07f 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -10120,14 +10120,10 @@ LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) { return Legalized; } - bool IsVolatile = MemOp->isVolatile(); - // Don't try to optimize volatile. - if (IsVolatile) - return UnableToLegalize; - if (MaxLen && KnownLen > MaxLen) return UnableToLegalize; + bool IsVolatile = MemOp->isVolatile(); if (Opc == TargetOpcode::G_MEMCPY) { auto &MF = *MI.getParent()->getParent(); const auto &TLI = *MF.getSubtarget().getTargetLowering(); diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp index d2b2edf..df162fc 100644 --- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp +++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp @@ -600,87 +600,113 @@ static Value *getMask(Value *WideMask, unsigned Factor, bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic( IntrinsicInst *DI, SmallSetVector<Instruction *, 32> &DeadInsts) { - Value *LoadedVal = DI->getOperand(0); - if (!LoadedVal->hasOneUse() || !isa<LoadInst, VPIntrinsic>(LoadedVal)) + Instruction *LoadedVal = dyn_cast<Instruction>(DI->getOperand(0)); + if (!LoadedVal || !LoadedVal->hasOneUse()) + return false; + + auto *LI = dyn_cast<LoadInst>(LoadedVal); + auto *II = dyn_cast<IntrinsicInst>(LoadedVal); + if (!LI && !II) return false; const unsigned Factor = getDeinterleaveIntrinsicFactor(DI->getIntrinsicID()); assert(Factor && "unexpected deinterleave intrinsic"); Value *Mask = nullptr; - if (auto *VPLoad = dyn_cast<VPIntrinsic>(LoadedVal)) { - if (VPLoad->getIntrinsicID() != Intrinsic::vp_load) + if (LI) { + if (!LI->isSimple()) return false; + + LLVM_DEBUG(dbgs() << "IA: Found a load with deinterleave intrinsic " << *DI + << " and factor = " << Factor << "\n"); + } else { + assert(II); + // Check mask operand. Handle both all-true/false and interleaved mask. - Value *WideMask = VPLoad->getOperand(1); - Mask = getMask(WideMask, Factor, getDeinterleavedVectorType(DI)); - if (!Mask) + Value *WideMask; + switch (II->getIntrinsicID()) { + default: return false; + case Intrinsic::vp_load: + WideMask = II->getOperand(1); + break; + case Intrinsic::masked_load: + WideMask = II->getOperand(2); + break; + } - LLVM_DEBUG(dbgs() << "IA: Found a vp.load with deinterleave intrinsic " - << *DI << " and factor = " << Factor << "\n"); - } else { - auto *LI = cast<LoadInst>(LoadedVal); - if (!LI->isSimple()) + Mask = getMask(WideMask, Factor, getDeinterleavedVectorType(DI)); + if (!Mask) return false; - LLVM_DEBUG(dbgs() << "IA: Found a load with deinterleave intrinsic " << *DI - << " and factor = " << Factor << "\n"); + LLVM_DEBUG(dbgs() << "IA: Found a vp.load or masked.load with deinterleave" + << " intrinsic " << *DI << " and factor = " + << Factor << "\n"); } // Try and match this with target specific intrinsics. - if (!TLI->lowerDeinterleaveIntrinsicToLoad(cast<Instruction>(LoadedVal), Mask, - DI)) + if (!TLI->lowerDeinterleaveIntrinsicToLoad(LoadedVal, Mask, DI)) return false; DeadInsts.insert(DI); // We now have a target-specific load, so delete the old one. - DeadInsts.insert(cast<Instruction>(LoadedVal)); + DeadInsts.insert(LoadedVal); return true; } bool InterleavedAccessImpl::lowerInterleaveIntrinsic( - IntrinsicInst *II, SmallSetVector<Instruction *, 32> &DeadInsts) { - if (!II->hasOneUse()) + IntrinsicInst *IntII, SmallSetVector<Instruction *, 32> &DeadInsts) { + if (!IntII->hasOneUse()) return false; - Value *StoredBy = II->user_back(); - if (!isa<StoreInst, VPIntrinsic>(StoredBy)) + Instruction *StoredBy = dyn_cast<Instruction>(IntII->user_back()); + if (!StoredBy) + return false; + auto *SI = dyn_cast<StoreInst>(StoredBy); + auto *II = dyn_cast<IntrinsicInst>(StoredBy); + if (!SI && !II) return false; - SmallVector<Value *, 8> InterleaveValues(II->args()); - const unsigned Factor = getInterleaveIntrinsicFactor(II->getIntrinsicID()); + SmallVector<Value *, 8> InterleaveValues(IntII->args()); + const unsigned Factor = getInterleaveIntrinsicFactor(IntII->getIntrinsicID()); assert(Factor && "unexpected interleave intrinsic"); Value *Mask = nullptr; - if (auto *VPStore = dyn_cast<VPIntrinsic>(StoredBy)) { - if (VPStore->getIntrinsicID() != Intrinsic::vp_store) + if (II) { + // Check mask operand. Handle both all-true/false and interleaved mask. + Value *WideMask; + switch (II->getIntrinsicID()) { + default: return false; - - Value *WideMask = VPStore->getOperand(2); + case Intrinsic::vp_store: + WideMask = II->getOperand(2); + break; + case Intrinsic::masked_store: + WideMask = II->getOperand(3); + break; + } Mask = getMask(WideMask, Factor, cast<VectorType>(InterleaveValues[0]->getType())); if (!Mask) return false; - LLVM_DEBUG(dbgs() << "IA: Found a vp.store with interleave intrinsic " - << *II << " and factor = " << Factor << "\n"); + LLVM_DEBUG(dbgs() << "IA: Found a vp.store or masked.store with interleave" + << " intrinsic " << *IntII << " and factor = " + << Factor << "\n"); } else { - auto *SI = cast<StoreInst>(StoredBy); if (!SI->isSimple()) return false; - LLVM_DEBUG(dbgs() << "IA: Found a store with interleave intrinsic " << *II - << " and factor = " << Factor << "\n"); + LLVM_DEBUG(dbgs() << "IA: Found a store with interleave intrinsic " + << *IntII << " and factor = " << Factor << "\n"); } // Try and match this with target specific intrinsics. - if (!TLI->lowerInterleaveIntrinsicToStore(cast<Instruction>(StoredBy), Mask, - InterleaveValues)) + if (!TLI->lowerInterleaveIntrinsicToStore(StoredBy, Mask, InterleaveValues)) return false; // We now have a target-specific store, so delete the old one. - DeadInsts.insert(cast<Instruction>(StoredBy)); - DeadInsts.insert(II); + DeadInsts.insert(StoredBy); + DeadInsts.insert(IntII); return true; } diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp index b38a4d1c..90005bd 100644 --- a/llvm/lib/CodeGen/MachinePipeliner.cpp +++ b/llvm/lib/CodeGen/MachinePipeliner.cpp @@ -4279,8 +4279,8 @@ void LoopCarriedEdges::modifySUnits(std::vector<SUnit> &SUnits, !TII->isGlobalMemoryObject(FromMI) && !TII->isGlobalMemoryObject(ToMI) && !isSuccOrder(From, To)) { SDep Pred = Dep; - Pred.setSUnit(Src); - Dst->addPred(Pred); + Pred.setSUnit(From); + To->addPred(Pred); } } } diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp index 76cba29..9d5c39c 100644 --- a/llvm/lib/CodeGen/MachineScheduler.cpp +++ b/llvm/lib/CodeGen/MachineScheduler.cpp @@ -771,24 +771,6 @@ static bool isSchedBoundary(MachineBasicBlock::iterator MI, MI->isFakeUse(); } -/// A region of an MBB for scheduling. -namespace { -struct SchedRegion { - /// RegionBegin is the first instruction in the scheduling region, and - /// RegionEnd is either MBB->end() or the scheduling boundary after the - /// last instruction in the scheduling region. These iterators cannot refer - /// to instructions outside of the identified scheduling region because - /// those may be reordered before scheduling this region. - MachineBasicBlock::iterator RegionBegin; - MachineBasicBlock::iterator RegionEnd; - unsigned NumRegionInstrs; - - SchedRegion(MachineBasicBlock::iterator B, MachineBasicBlock::iterator E, - unsigned N) : - RegionBegin(B), RegionEnd(E), NumRegionInstrs(N) {} -}; -} // end anonymous namespace - using MBBRegionsVector = SmallVector<SchedRegion, 16>; static void @@ -3725,7 +3707,8 @@ void GenericScheduler::initPolicy(MachineBasicBlock::iterator Begin, RegionPolicy.OnlyBottomUp = true; // Allow the subtarget to override default policy. - MF.getSubtarget().overrideSchedPolicy(RegionPolicy, NumRegionInstrs); + SchedRegion Region(Begin, End, NumRegionInstrs); + MF.getSubtarget().overrideSchedPolicy(RegionPolicy, Region); // After subtarget overrides, apply command line options. if (!EnableRegPressure) { @@ -4338,7 +4321,8 @@ void PostGenericScheduler::initPolicy(MachineBasicBlock::iterator Begin, RegionPolicy.OnlyBottomUp = false; // Allow the subtarget to override default policy. - MF.getSubtarget().overridePostRASchedPolicy(RegionPolicy, NumRegionInstrs); + SchedRegion Region(Begin, End, NumRegionInstrs); + MF.getSubtarget().overridePostRASchedPolicy(RegionPolicy, Region); // After subtarget overrides, apply command line options. if (PostRADirection == MISched::TopDown) { diff --git a/llvm/lib/CodeGen/RegAllocBasic.cpp b/llvm/lib/CodeGen/RegAllocBasic.cpp index 381249e..0b2a73b 100644 --- a/llvm/lib/CodeGen/RegAllocBasic.cpp +++ b/llvm/lib/CodeGen/RegAllocBasic.cpp @@ -5,35 +5,31 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// -// This file defines the RABasic function pass, which provides a minimal -// implementation of the basic register allocator. -// +/// +/// \file +/// This file defines the RABasic function pass, which provides a minimal +/// implementation of the basic register allocator. +/// //===----------------------------------------------------------------------===// +#include "RegAllocBasic.h" #include "AllocationOrder.h" -#include "RegAllocBase.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/CodeGen/CalcSpillWeights.h" #include "llvm/CodeGen/LiveDebugVariables.h" #include "llvm/CodeGen/LiveIntervals.h" -#include "llvm/CodeGen/LiveRangeEdit.h" #include "llvm/CodeGen/LiveRegMatrix.h" #include "llvm/CodeGen/LiveStacks.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" -#include "llvm/CodeGen/Spiller.h" -#include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/VirtRegMap.h" #include "llvm/Pass.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include <queue> using namespace llvm; @@ -42,89 +38,8 @@ using namespace llvm; static RegisterRegAlloc basicRegAlloc("basic", "basic register allocator", createBasicRegisterAllocator); -namespace { - struct CompSpillWeight { - bool operator()(const LiveInterval *A, const LiveInterval *B) const { - return A->weight() < B->weight(); - } - }; -} - -namespace { -/// RABasic provides a minimal implementation of the basic register allocation -/// algorithm. It prioritizes live virtual registers by spill weight and spills -/// whenever a register is unavailable. This is not practical in production but -/// provides a useful baseline both for measuring other allocators and comparing -/// the speed of the basic algorithm against other styles of allocators. -class RABasic : public MachineFunctionPass, - public RegAllocBase, - private LiveRangeEdit::Delegate { - // context - MachineFunction *MF = nullptr; - - // state - std::unique_ptr<Spiller> SpillerInstance; - std::priority_queue<const LiveInterval *, std::vector<const LiveInterval *>, - CompSpillWeight> - Queue; - - // Scratch space. Allocated here to avoid repeated malloc calls in - // selectOrSplit(). - BitVector UsableRegs; - - bool LRE_CanEraseVirtReg(Register) override; - void LRE_WillShrinkVirtReg(Register) override; - -public: - RABasic(const RegAllocFilterFunc F = nullptr); - - /// Return the pass name. - StringRef getPassName() const override { return "Basic Register Allocator"; } - - /// RABasic analysis usage. - void getAnalysisUsage(AnalysisUsage &AU) const override; - - void releaseMemory() override; - - Spiller &spiller() override { return *SpillerInstance; } - - void enqueueImpl(const LiveInterval *LI) override { Queue.push(LI); } - - const LiveInterval *dequeue() override { - if (Queue.empty()) - return nullptr; - const LiveInterval *LI = Queue.top(); - Queue.pop(); - return LI; - } - - MCRegister selectOrSplit(const LiveInterval &VirtReg, - SmallVectorImpl<Register> &SplitVRegs) override; - - /// Perform register allocation. - bool runOnMachineFunction(MachineFunction &mf) override; - - MachineFunctionProperties getRequiredProperties() const override { - return MachineFunctionProperties().setNoPHIs(); - } - - MachineFunctionProperties getClearedProperties() const override { - return MachineFunctionProperties().setIsSSA(); - } - - // Helper for spilling all live virtual registers currently unified under preg - // that interfere with the most recently queried lvr. Return true if spilling - // was successful, and append any new spilled/split intervals to splitLVRs. - bool spillInterferences(const LiveInterval &VirtReg, MCRegister PhysReg, - SmallVectorImpl<Register> &SplitVRegs); - - static char ID; -}; - char RABasic::ID = 0; -} // end anonymous namespace - char &llvm::RABasicID = RABasic::ID; INITIALIZE_PASS_BEGIN(RABasic, "regallocbasic", "Basic Register Allocator", diff --git a/llvm/lib/CodeGen/RegAllocBasic.h b/llvm/lib/CodeGen/RegAllocBasic.h new file mode 100644 index 0000000..004bc1a --- /dev/null +++ b/llvm/lib/CodeGen/RegAllocBasic.h @@ -0,0 +1,104 @@ +//===-- RegAllocBasic.h - Basic Register Allocator Header -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file declares the RABasic class, which provides a minimal +/// implementation of the basic register allocator. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_REGALLOCBASIC_H +#define LLVM_CODEGEN_REGALLOCBASIC_H + +#include "RegAllocBase.h" +#include "llvm/CodeGen/LiveRangeEdit.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/Spiller.h" +#include <queue> + +namespace llvm { + +struct CompSpillWeight { + bool operator()(const LiveInterval *A, const LiveInterval *B) const { + return A->weight() < B->weight(); + } +}; + +/// RABasic provides a minimal implementation of the basic register allocation +/// algorithm. It prioritizes live virtual registers by spill weight and spills +/// whenever a register is unavailable. This is not practical in production but +/// provides a useful baseline both for measuring other allocators and comparing +/// the speed of the basic algorithm against other styles of allocators. +class LLVM_LIBRARY_VISIBILITY RABasic : public MachineFunctionPass, + public RegAllocBase, + private LiveRangeEdit::Delegate { + // context + MachineFunction *MF = nullptr; + + // state + std::unique_ptr<Spiller> SpillerInstance; + std::priority_queue<const LiveInterval *, std::vector<const LiveInterval *>, + CompSpillWeight> + Queue; + + // Scratch space. Allocated here to avoid repeated malloc calls in + // selectOrSplit(). + BitVector UsableRegs; + + bool LRE_CanEraseVirtReg(Register) override; + void LRE_WillShrinkVirtReg(Register) override; + +public: + RABasic(const RegAllocFilterFunc F = nullptr); + + /// Return the pass name. + StringRef getPassName() const override { return "Basic Register Allocator"; } + + /// RABasic analysis usage. + void getAnalysisUsage(AnalysisUsage &AU) const override; + + void releaseMemory() override; + + Spiller &spiller() override { return *SpillerInstance; } + + void enqueueImpl(const LiveInterval *LI) override { Queue.push(LI); } + + const LiveInterval *dequeue() override { + if (Queue.empty()) + return nullptr; + const LiveInterval *LI = Queue.top(); + Queue.pop(); + return LI; + } + + MCRegister selectOrSplit(const LiveInterval &VirtReg, + SmallVectorImpl<Register> &SplitVRegs) override; + + /// Perform register allocation. + bool runOnMachineFunction(MachineFunction &mf) override; + + MachineFunctionProperties getRequiredProperties() const override { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::NoPHIs); + } + + MachineFunctionProperties getClearedProperties() const override { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::IsSSA); + } + + // Helper for spilling all live virtual registers currently unified under preg + // that interfere with the most recently queried lvr. Return true if spilling + // was successful, and append any new spilled/split intervals to splitLVRs. + bool spillInterferences(const LiveInterval &VirtReg, MCRegister PhysReg, + SmallVectorImpl<Register> &SplitVRegs); + + static char ID; +}; +} // namespace llvm +#endif diff --git a/llvm/lib/CodeGen/SafeStack.cpp b/llvm/lib/CodeGen/SafeStack.cpp index 9962070..908ed96 100644 --- a/llvm/lib/CodeGen/SafeStack.cpp +++ b/llvm/lib/CodeGen/SafeStack.cpp @@ -614,6 +614,13 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack( Use &U = *AI->use_begin(); Instruction *User = cast<Instruction>(U.getUser()); + // Drop lifetime markers now that this is no longer an alloca. + // SafeStack has already performed its own stack coloring. + if (User->isLifetimeStartOrEnd()) { + User->eraseFromParent(); + continue; + } + Instruction *InsertBefore; if (auto *PHI = dyn_cast<PHINode>(User)) InsertBefore = PHI->getIncomingBlock(U)->getTerminator(); diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index fed5e72..d3df434 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -12375,11 +12375,8 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))) { // Any flags available in a select/setcc fold will be on the setcc as they // migrated from fcmp - Flags = N0->getFlags(); - SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, - N2, N0.getOperand(2)); - SelectNode->setFlags(Flags); - return SelectNode; + return DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, N2, + N0.getOperand(2), N0->getFlags()); } if (SDValue ABD = foldSelectToABD(Cond0, Cond1, N1, N2, CC, DL)) @@ -16738,7 +16735,8 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) { // Fold freeze(op(x, ...)) -> op(freeze(x), ...). // Try to push freeze through instructions that propagate but don't produce // poison as far as possible. If an operand of freeze follows three - // conditions 1) one-use, and 2) does not produce poison then push + // conditions 1) one-use, 2) does not produce poison, and 3) has all but one + // guaranteed-non-poison operands (or is a BUILD_VECTOR or similar) then push // the freeze through to the operands that are not guaranteed non-poison. // NOTE: we will strip poison-generating flags, so ignore them here. if (DAG.canCreateUndefOrPoison(N0, /*PoisonOnly*/ false, @@ -16746,6 +16744,18 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) { N0->getNumValues() != 1 || !N0->hasOneUse()) return SDValue(); + // TOOD: we should always allow multiple operands, however this increases the + // likelihood of infinite loops due to the ReplaceAllUsesOfValueWith call + // below causing later nodes that share frozen operands to fold again and no + // longer being able to confirm other operands are not poison due to recursion + // depth limits on isGuaranteedNotToBeUndefOrPoison. + bool AllowMultipleMaybePoisonOperands = + N0.getOpcode() == ISD::SELECT_CC || N0.getOpcode() == ISD::SETCC || + N0.getOpcode() == ISD::BUILD_VECTOR || + N0.getOpcode() == ISD::BUILD_PAIR || + N0.getOpcode() == ISD::VECTOR_SHUFFLE || + N0.getOpcode() == ISD::CONCAT_VECTORS || N0.getOpcode() == ISD::FMUL; + // Avoid turning a BUILD_VECTOR that can be recognized as "all zeros", "all // ones" or "constant" into something that depends on FrozenUndef. We can // instead pick undef values to keep those properties, while at the same time @@ -16772,8 +16782,16 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) { if (DAG.isGuaranteedNotToBeUndefOrPoison(Op, /*PoisonOnly*/ false, /*Depth*/ 1)) continue; - if (MaybePoisonOperands.insert(Op).second) + bool HadMaybePoisonOperands = !MaybePoisonOperands.empty(); + bool IsNewMaybePoisonOperand = MaybePoisonOperands.insert(Op).second; + if (IsNewMaybePoisonOperand) MaybePoisonOperandNumbers.push_back(OpNo); + if (!HadMaybePoisonOperands) + continue; + if (IsNewMaybePoisonOperand && !AllowMultipleMaybePoisonOperands) { + // Multiple maybe-poison ops when not allowed - bail out. + return SDValue(); + } } // NOTE: the whole op may be not guaranteed to not be undef or poison because // it could create undef or poison due to it's poison-generating flags. @@ -22727,11 +22745,7 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { SDValue DAGCombiner::visitLIFETIME_END(SDNode *N) { const auto *LifetimeEnd = cast<LifetimeSDNode>(N); - if (!LifetimeEnd->hasOffset()) - return SDValue(); - - const BaseIndexOffset LifetimeEndBase(N->getOperand(1), SDValue(), - LifetimeEnd->getOffset(), false); + const BaseIndexOffset LifetimeEndBase(N->getOperand(1), SDValue(), 0, false); // We walk up the chains to find stores. SmallVector<SDValue, 8> Chains = {N->getOperand(0)}; @@ -29418,9 +29432,8 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { return {false /*isVolatile*/, /*isAtomic*/ false, LN->getOperand(1), - (LN->hasOffset()) ? LN->getOffset() : 0, - (LN->hasOffset()) ? LocationSize::precise(LN->getSize()) - : LocationSize::beforeOrAfterPointer(), + 0, + LocationSize::precise(LN->getSize()), (MachineMemOperand *)nullptr}; // Default. return {false /*isvolatile*/, diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 7266940..74172b2 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -2785,19 +2785,17 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node, // In strict mode, we must avoid spurious exceptions, and therefore // must make sure to only emit a single STRICT_SINT_TO_FP. SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0); - Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, - { Node->getOperand(0), InCvt }); - Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, - { Fast.getValue(1), Fast, Fast }); - Chain = Slow.getValue(1); // The STRICT_SINT_TO_FP inherits the exception mode from the // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can // never raise any exception. SDNodeFlags Flags; Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept()); - Fast->setFlags(Flags); + Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {DestVT, MVT::Other}, + {Node->getOperand(0), InCvt}, Flags); Flags.setNoFPExcept(true); - Slow->setFlags(Flags); + Slow = DAG.getNode(ISD::STRICT_FADD, dl, {DestVT, MVT::Other}, + {Fast.getValue(1), Fast, Fast}, Flags); + Chain = Slow.getValue(1); } else { SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or); Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt); @@ -3407,14 +3405,12 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { EVT VT = Operand.getValueType(); SDValue One = DAG.getConstantFP(1.0, dl, VT); SDValue Chain = DAG.getEntryNode(); - SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, dl, {VT, MVT::Other}, - {Chain, Operand, One}); - // Propagate existing flags on canonicalize, and additionally set // NoFPExcept. SDNodeFlags CanonicalizeFlags = Node->getFlags(); CanonicalizeFlags.setNoFPExcept(true); - Mul->setFlags(CanonicalizeFlags); + SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, dl, {VT, MVT::Other}, + {Chain, Operand, One}, CanonicalizeFlags); Results.push_back(Mul); break; @@ -4150,15 +4146,14 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { Tmp2 = Node->getOperand(1); Tmp3 = Node->getOperand(2); if (Tmp1.getOpcode() == ISD::SETCC) { - Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), - Tmp2, Tmp3, - cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); + Tmp1 = DAG.getSelectCC( + dl, Tmp1.getOperand(0), Tmp1.getOperand(1), Tmp2, Tmp3, + cast<CondCodeSDNode>(Tmp1.getOperand(2))->get(), Node->getFlags()); } else { - Tmp1 = DAG.getSelectCC(dl, Tmp1, - DAG.getConstant(0, dl, Tmp1.getValueType()), - Tmp2, Tmp3, ISD::SETNE); + Tmp1 = + DAG.getSelectCC(dl, Tmp1, DAG.getConstant(0, dl, Tmp1.getValueType()), + Tmp2, Tmp3, ISD::SETNE, Node->getFlags()); } - Tmp1->setFlags(Node->getFlags()); Results.push_back(Tmp1); break; case ISD::BR_JT: { @@ -4296,8 +4291,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { EVT Tmp1VT = Tmp1.getValueType(); Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, DAG.getBoolConstant(true, dl, VT, Tmp1VT), - DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3); - Tmp1->setFlags(Node->getFlags()); + DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3, + Node->getFlags()); Results.push_back(Tmp1); break; } @@ -4335,8 +4330,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) { // Use the new condition code and swap true and false Legalized = true; - Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); - Tmp1->setFlags(Node->getFlags()); + Tmp1 = + DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC, Node->getFlags()); } else { // If The inverse is not legal, then try to swap the arguments using // the inverse condition code. @@ -4345,8 +4340,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { // The swapped inverse condition is legal, so swap true and false, // lhs and rhs. Legalized = true; - Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); - Tmp1->setFlags(Node->getFlags()); + Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC, + Node->getFlags()); } } @@ -4365,15 +4360,14 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { // If we expanded the SETCC by swapping LHS and RHS, or by inverting the // condition code, create a new SELECT_CC node. if (CC.getNode()) { - Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), - Tmp1, Tmp2, Tmp3, Tmp4, CC); + Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, + Tmp2, Tmp3, Tmp4, CC, Node->getFlags()); } else { Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType()); CC = DAG.getCondCode(ISD::SETNE); Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, - Tmp2, Tmp3, Tmp4, CC); + Tmp2, Tmp3, Tmp4, CC, Node->getFlags()); } - Tmp1->setFlags(Node->getFlags()); } Results.push_back(Tmp1); break; diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index f908a66..d2ecc133 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -2087,11 +2087,10 @@ void VectorLegalizer::ExpandSETCC(SDNode *Node, // Otherwise, SETCC for the given comparison type must be completely // illegal; expand it into a SELECT_CC. EVT VT = Node->getValueType(0); - LHS = - DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS, - DAG.getBoolConstant(true, dl, VT, LHS.getValueType()), - DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), CC); - LHS->setFlags(Node->getFlags()); + LHS = DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS, + DAG.getBoolConstant(true, dl, VT, LHS.getValueType()), + DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), + CC, Node->getFlags()); } Results.push_back(LHS); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 32c5961..1661814 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -372,9 +372,9 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N, SDVTList ScalarVTs = DAG.getVTList( ResVT.getVectorElementType(), OvVT.getVectorElementType()); - SDNode *ScalarNode = DAG.getNode( - N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode(); - ScalarNode->setFlags(N->getFlags()); + SDNode *ScalarNode = DAG.getNode(N->getOpcode(), DL, ScalarVTs, + {ScalarLHS, ScalarRHS}, N->getFlags()) + .getNode(); // Replace the other vector result not being explicitly scalarized here. unsigned OtherNo = 1 - ResNo; @@ -1898,7 +1898,7 @@ SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) { NE = ResNE; //The results of each unrolled operation, including the chain. - EVT ChainVTs[] = {EltVT, MVT::Other}; + SDVTList ChainVTs = DAG.getVTList(EltVT, MVT::Other); SmallVector<SDValue, 8> Chains; unsigned i; @@ -1914,8 +1914,8 @@ SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) { Operands[j] = Operand; } } - SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands); - Scalar.getNode()->setFlags(N->getFlags()); + SDValue Scalar = + DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands, N->getFlags()); //Add in the scalar as well as its chain value to the //result vectors. @@ -1956,10 +1956,10 @@ void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo, unsigned Opcode = N->getOpcode(); SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT); SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT); - SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode(); - SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode(); - LoNode->setFlags(N->getFlags()); - HiNode->setFlags(N->getFlags()); + SDNode *LoNode = + DAG.getNode(Opcode, dl, LoVTs, {LoLHS, LoRHS}, N->getFlags()).getNode(); + SDNode *HiNode = + DAG.getNode(Opcode, dl, HiVTs, {HiLHS, HiRHS}, N->getFlags()).getNode(); Lo = SDValue(LoNode, ResNo); Hi = SDValue(HiNode, ResNo); @@ -2669,10 +2669,8 @@ void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N, else std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); - Lo = DAG.getNode(N->getOpcode(), dl, {LoVT, LoVT1}, Lo); - Hi = DAG.getNode(N->getOpcode(), dl, {HiVT, HiVT1}, Hi); - Lo->setFlags(N->getFlags()); - Hi->setFlags(N->getFlags()); + Lo = DAG.getNode(N->getOpcode(), dl, {LoVT, LoVT1}, Lo, N->getFlags()); + Hi = DAG.getNode(N->getOpcode(), dl, {HiVT, HiVT1}, Hi, N->getFlags()); SDNode *HiNode = Hi.getNode(); SDNode *LoNode = Lo.getNode(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 2458115..773ff48 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -786,10 +786,7 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { break; case ISD::LIFETIME_START: case ISD::LIFETIME_END: - if (cast<LifetimeSDNode>(N)->hasOffset()) { - ID.AddInteger(cast<LifetimeSDNode>(N)->getSize()); - ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset()); - } + ID.AddInteger(cast<LifetimeSDNode>(N)->getSize()); break; case ISD::PSEUDO_PROBE: ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid()); @@ -3036,7 +3033,7 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, *this, Depth); break; -} + } // We don't support other cases than those above for scalable vectors at // the moment. @@ -9364,7 +9361,7 @@ SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, - int64_t Size, int64_t Offset) { + int64_t Size) { const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END; const auto VTs = getVTList(MVT::Other); SDValue Ops[2] = { @@ -9377,13 +9374,12 @@ SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl, AddNodeIDNode(ID, Opcode, VTs, Ops); ID.AddInteger(FrameIndex); ID.AddInteger(Size); - ID.AddInteger(Offset); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) return SDValue(E, 0); - LifetimeSDNode *N = newSDNode<LifetimeSDNode>( - Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset); + LifetimeSDNode *N = newSDNode<LifetimeSDNode>(Opcode, dl.getIROrder(), + dl.getDebugLoc(), VTs, Size); createOperands(N, Ops); CSEMap.InsertNode(N, IP); InsertNode(N); @@ -10563,7 +10559,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef<SDUse> Ops) { switch (Ops.size()) { case 0: return getNode(Opcode, DL, VT); - case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0])); + case 1: return getNode(Opcode, DL, VT, Ops[0].get()); case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); default: break; @@ -10699,7 +10695,16 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) { - return getNode(Opcode, DL, getVTList(ResultTys), Ops); + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNode(Opcode, DL, getVTList(ResultTys), Ops, Flags); +} + +SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, + ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops, + const SDNodeFlags Flags) { + return getNode(Opcode, DL, getVTList(ResultTys), Ops, Flags); } SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, @@ -10855,26 +10860,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) && "Invalid STRICT_FP_ROUND!"); break; -#if 0 - // FIXME: figure out how to safely handle things like - // int foo(int x) { return 1 << (x & 255); } - // int bar() { return foo(256); } - case ISD::SRA_PARTS: - case ISD::SRL_PARTS: - case ISD::SHL_PARTS: - if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && - cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) - return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); - else if (N3.getOpcode() == ISD::AND) - if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { - // If the and is only masking out bits that cannot effect the shift, - // eliminate the and. - unsigned NumBits = VT.getScalarSizeInBits()*2; - if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) - return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); - } - break; -#endif } // Memoize the node unless it returns a glue result. diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp index da92aaa..8f08046 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp @@ -303,10 +303,7 @@ BaseIndexOffset BaseIndexOffset::match(const SDNode *N, if (const auto *LS0 = dyn_cast<LSBaseSDNode>(N)) return matchLSNode(LS0, DAG); if (const auto *LN = dyn_cast<LifetimeSDNode>(N)) { - if (LN->hasOffset()) - return BaseIndexOffset(LN->getOperand(1), SDValue(), LN->getOffset(), - false); - return BaseIndexOffset(LN->getOperand(1), SDValue(), false); + return BaseIndexOffset(LN->getOperand(1), SDValue(), 0, false); } return BaseIndexOffset(); } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 01e5312..1636465 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -7596,32 +7596,17 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, const int64_t ObjectSize = cast<ConstantInt>(I.getArgOperand(0))->getSExtValue(); - Value *const ObjectPtr = I.getArgOperand(1); - SmallVector<const Value *, 4> Allocas; - getUnderlyingObjects(ObjectPtr, Allocas); + const AllocaInst *LifetimeObject = cast<AllocaInst>(I.getArgOperand(1)); - for (const Value *Alloca : Allocas) { - const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca); - - // Could not find an Alloca. - if (!LifetimeObject) - continue; - - // First check that the Alloca is static, otherwise it won't have a - // valid frame index. - auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject); - if (SI == FuncInfo.StaticAllocaMap.end()) - return; + // First check that the Alloca is static, otherwise it won't have a + // valid frame index. + auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject); + if (SI == FuncInfo.StaticAllocaMap.end()) + return; - const int FrameIndex = SI->second; - int64_t Offset; - if (GetPointerBaseWithConstantOffset( - ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject) - Offset = -1; // Cannot determine offset from alloca to lifetime object. - Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize, - Offset); - DAG.setRoot(Res); - } + const int FrameIndex = SI->second; + Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize); + DAG.setRoot(Res); return; } case Intrinsic::pseudoprobe: { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index 7fc1558..9474587 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -947,8 +947,7 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { << ASC->getDestAddressSpace() << ']'; } else if (const LifetimeSDNode *LN = dyn_cast<LifetimeSDNode>(this)) { - if (LN->hasOffset()) - OS << "<" << LN->getOffset() << " to " << LN->getOffset() + LN->getSize() << ">"; + OS << "<0 to " << LN->getSize() << ">"; } else if (const auto *AA = dyn_cast<AssertAlignSDNode>(this)) { OS << '<' << AA->getAlign().value() << '>'; } diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index e059798..1764910 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -778,7 +778,7 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( case ISD::FREEZE: { SDValue N0 = Op.getOperand(0); if (DAG.isGuaranteedNotToBeUndefOrPoison(N0, DemandedElts, - /*PoisonOnly=*/false)) + /*PoisonOnly=*/false, Depth + 1)) return N0; break; } @@ -3369,7 +3369,8 @@ bool TargetLowering::SimplifyDemandedVectorElts( case ISD::FREEZE: { SDValue N0 = Op.getOperand(0); if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(N0, DemandedElts, - /*PoisonOnly=*/false)) + /*PoisonOnly=*/false, + Depth + 1)) return TLO.CombineTo(Op, N0); // TODO: Replace this with the general fold from DAGCombiner::visitFREEZE @@ -8128,7 +8129,7 @@ static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) { return ISD::matchUnaryPredicate( Z, [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; }, - /*AllowUndef=*/true, /*AllowTruncation=*/true); + /*AllowUndefs=*/true, /*AllowTruncation=*/true); } static SDValue expandVPFunnelShift(SDNode *Node, SelectionDAG &DAG) { @@ -8633,9 +8634,8 @@ TargetLowering::createSelectForFMINNUM_FMAXNUM(SDNode *Node, return SDValue(); SDValue Op1 = Node->getOperand(0); SDValue Op2 = Node->getOperand(1); - SDValue SelCC = DAG.getSelectCC(SDLoc(Node), Op1, Op2, Op1, Op2, Pred); - SelCC->setFlags(Node->getFlags()); - return SelCC; + return DAG.getSelectCC(SDLoc(Node), Op1, Op2, Op1, Op2, Pred, + Node->getFlags()); } return SDValue(); @@ -11994,8 +11994,7 @@ SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node, // Get the mask value and add it to the current output position. This // either increments by 1 if MaskI is true or adds 0 otherwise. // Freeze in case we have poison/undef mask entries. - SDValue MaskI = - DAG.getFreeze(DAG.getExtractVectorElt(DL, MaskScalarVT, Mask, I)); + SDValue MaskI = DAG.getExtractVectorElt(DL, MaskScalarVT, Mask, I); MaskI = DAG.getFreeze(MaskI); MaskI = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, MaskI); MaskI = DAG.getNode(ISD::ZERO_EXTEND, DL, PositionVT, MaskI); |