diff options
Diffstat (limited to 'llvm/lib/CodeGen')
26 files changed, 296 insertions, 246 deletions
diff --git a/llvm/lib/CodeGen/AsmPrinter/AIXException.cpp b/llvm/lib/CodeGen/AsmPrinter/AIXException.cpp index 5d7c97a..6356d71 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AIXException.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AIXException.cpp @@ -37,8 +37,8 @@ void AIXException::emitExceptionInfoTable(const MCSymbol *LSDA, // unsigned long personality; /* Pointer to the personality routine */ // } - auto *EHInfo = - cast<MCSectionXCOFF>(Asm->getObjFileLowering().getCompactUnwindSection()); + auto *EHInfo = static_cast<MCSectionXCOFF *>( + Asm->getObjFileLowering().getCompactUnwindSection()); if (Asm->TM.getFunctionSections()) { // If option -ffunction-sections is on, append the function name to the // name of EH Info Table csect so that each function has its own EH Info diff --git a/llvm/lib/CodeGen/AsmPrinter/ARMException.cpp b/llvm/lib/CodeGen/AsmPrinter/ARMException.cpp index de6ebcf..51342c6 100644 --- a/llvm/lib/CodeGen/AsmPrinter/ARMException.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/ARMException.cpp @@ -39,7 +39,7 @@ void ARMException::beginFunction(const MachineFunction *MF) { if (CFISecType == AsmPrinter::CFISection::Debug) { if (!hasEmittedCFISections) { if (Asm->getModuleCFISectionType() == AsmPrinter::CFISection::Debug) - Asm->OutStreamer->emitCFISections(false, true); + Asm->OutStreamer->emitCFISections(false, true, false); hasEmittedCFISections = true; } diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 10bdb81..6166271 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1868,6 +1868,7 @@ void AsmPrinter::emitFunctionBody() { OutStreamer->emitLabel(MI.getOperand(0).getMCSymbol()); break; case TargetOpcode::EH_LABEL: + OutStreamer->AddComment("EH_LABEL"); OutStreamer->emitLabel(MI.getOperand(0).getMCSymbol()); // For AsynchEH, insert a Nop if followed by a trap inst // Or the exception won't be caught. @@ -4220,10 +4221,11 @@ MCSymbol *AsmPrinter::GetCPISymbol(unsigned CPID) const { SectionKind Kind = CPE.getSectionKind(&DL); const Constant *C = CPE.Val.ConstVal; Align Alignment = CPE.Alignment; - if (const MCSectionCOFF *S = dyn_cast<MCSectionCOFF>( - getObjFileLowering().getSectionForConstant(DL, Kind, C, - Alignment))) { - if (MCSymbol *Sym = S->getCOMDATSymbol()) { + auto *S = + getObjFileLowering().getSectionForConstant(DL, Kind, C, Alignment); + if (S && TM.getTargetTriple().isOSBinFormatCOFF()) { + if (MCSymbol *Sym = + static_cast<const MCSectionCOFF *>(S)->getCOMDATSymbol()) { if (Sym->isUndefined()) OutStreamer->emitSymbolAttribute(Sym, MCSA_Global); return Sym; diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp index 8abeb56..c5d6e40 100644 --- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp @@ -1051,10 +1051,10 @@ void CodeViewDebug::switchToDebugSectionForSymbol(const MCSymbol *GVSym) { // comdat key. A section may be comdat because of -ffunction-sections or // because it is comdat in the IR. MCSectionCOFF *GVSec = - GVSym ? dyn_cast<MCSectionCOFF>(&GVSym->getSection()) : nullptr; + GVSym ? static_cast<MCSectionCOFF *>(&GVSym->getSection()) : nullptr; const MCSymbol *KeySym = GVSec ? GVSec->getCOMDATSymbol() : nullptr; - MCSectionCOFF *DebugSec = cast<MCSectionCOFF>( + auto *DebugSec = static_cast<MCSectionCOFF *>( CompilerInfoAsm->getObjFileLowering().getCOFFDebugSymbolsSection()); DebugSec = OS.getContext().getAssociativeCOFFSection(DebugSec, KeySym); diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp index 4fac4bb..6b8d08c 100644 --- a/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp @@ -109,9 +109,11 @@ void DwarfCFIException::beginBasicBlockSection(const MachineBasicBlock &MBB) { // chose not to be verbose in that case. And with `ForceDwarfFrameSection`, // we should always emit .debug_frame. if (CFISecType == AsmPrinter::CFISection::Debug || - Asm->TM.Options.ForceDwarfFrameSection) + Asm->TM.Options.ForceDwarfFrameSection || + Asm->TM.Options.MCOptions.EmitSFrameUnwind) Asm->OutStreamer->emitCFISections( - CFISecType == AsmPrinter::CFISection::EH, true); + CFISecType == AsmPrinter::CFISection::EH, true, + Asm->TM.Options.MCOptions.EmitSFrameUnwind); hasEmittedCFISections = true; } diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp index 8e8cda4..5577a7d 100644 --- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp @@ -1379,7 +1379,7 @@ void DwarfCompileUnit::constructCallSiteParmEntryDIEs( DIE *DwarfCompileUnit::constructImportedEntityDIE( const DIImportedEntity *Module) { - DIE *IMDie = DIE::get(DIEValueAllocator, (dwarf::Tag)Module->getTag()); + DIE *IMDie = DIE::get(DIEValueAllocator, Module->getTag()); insertDIE(Module, IMDie); DIE *EntityDie; auto *Entity = Module->getEntity(); diff --git a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp index dccd71f..13fd270 100644 --- a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp @@ -323,12 +323,6 @@ const MCExpr *WinException::getLabel(const MCSymbol *Label) { Asm->OutContext); } -const MCExpr *WinException::getLabelPlusOne(const MCSymbol *Label) { - return MCBinaryExpr::createAdd(getLabel(Label), - MCConstantExpr::create(1, Asm->OutContext), - Asm->OutContext); -} - const MCExpr *WinException::getOffset(const MCSymbol *OffsetOf, const MCSymbol *OffsetFrom) { return MCBinaryExpr::createSub( @@ -655,7 +649,7 @@ void WinException::emitSEHActionsForRange(const WinEHFuncInfo &FuncInfo, AddComment("LabelStart"); OS.emitValue(getLabel(BeginLabel), 4); AddComment("LabelEnd"); - OS.emitValue(getLabelPlusOne(EndLabel), 4); + OS.emitValue(getLabel(EndLabel), 4); AddComment(UME.IsFinally ? "FinallyFunclet" : UME.Filter ? "FilterFunction" : "CatchAll"); OS.emitValue(FilterOrFinally, 4); @@ -950,13 +944,7 @@ void WinException::computeIP2StateTable( if (!ChangeLabel) ChangeLabel = StateChange.PreviousEndLabel; // Emit an entry indicating that PCs after 'Label' have this EH state. - // NOTE: On ARM architectures, the StateFromIp automatically takes into - // account that the return address is after the call instruction (whose EH - // state we should be using), but on other platforms we need to +1 to the - // label so that we are using the correct EH state. - const MCExpr *LabelExpression = (isAArch64 || isThumb) - ? getLabel(ChangeLabel) - : getLabelPlusOne(ChangeLabel); + const MCExpr *LabelExpression = getLabel(ChangeLabel); IPToStateTable.push_back( std::make_pair(LabelExpression, StateChange.NewState)); // FIXME: assert that NewState is between CatchLow and CatchHigh. diff --git a/llvm/lib/CodeGen/AsmPrinter/WinException.h b/llvm/lib/CodeGen/AsmPrinter/WinException.h index 638589a..47dd30c 100644 --- a/llvm/lib/CodeGen/AsmPrinter/WinException.h +++ b/llvm/lib/CodeGen/AsmPrinter/WinException.h @@ -80,7 +80,6 @@ class LLVM_LIBRARY_VISIBILITY WinException : public EHStreamer { const MCExpr *create32bitRef(const MCSymbol *Value); const MCExpr *create32bitRef(const GlobalValue *GV); const MCExpr *getLabel(const MCSymbol *Label); - const MCExpr *getLabelPlusOne(const MCSymbol *Label); const MCExpr *getOffset(const MCSymbol *OffsetOf, const MCSymbol *OffsetFrom); const MCExpr *getOffsetPlusOne(const MCSymbol *OffsetOf, const MCSymbol *OffsetFrom); diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp index c3b4077..989cf4c4 100644 --- a/llvm/lib/CodeGen/CodeGen.cpp +++ b/llvm/lib/CodeGen/CodeGen.cpp @@ -45,7 +45,6 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializeExpandPostRALegacyPass(Registry); initializeFEntryInserterLegacyPass(Registry); initializeFinalizeISelPass(Registry); - initializeFinalizeMachineBundlesPass(Registry); initializeFixupStatepointCallerSavedLegacyPass(Registry); initializeFuncletLayoutPass(Registry); initializeGCMachineCodeAnalysisPass(Registry); diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index c21058c..416c56d 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -2095,6 +2095,10 @@ static bool isRemOfLoopIncrementWithLoopInvariant( if (!L->isLoopInvariant(RemAmt)) return false; + // Only works if the AddOffset is a loop invaraint + if (AddOffset && !L->isLoopInvariant(AddOffset)) + return false; + // Is the PHI a loop increment? auto LoopIncrInfo = getIVIncrement(PN, LI); if (!LoopIncrInfo) diff --git a/llvm/lib/CodeGen/ExpandFp.cpp b/llvm/lib/CodeGen/ExpandFp.cpp index 714ec55..1c1047c 100644 --- a/llvm/lib/CodeGen/ExpandFp.cpp +++ b/llvm/lib/CodeGen/ExpandFp.cpp @@ -103,10 +103,10 @@ static void expandFPToI(Instruction *FPToI) { Value *A1 = nullptr; if (FloatVal->getType()->isHalfTy()) { if (FPToI->getOpcode() == Instruction::FPToUI) { - Value *A0 = Builder.CreateFPToUI(FloatVal, Builder.getIntNTy(32)); + Value *A0 = Builder.CreateFPToUI(FloatVal, Builder.getInt32Ty()); A1 = Builder.CreateZExt(A0, IntTy); } else { // FPToSI - Value *A0 = Builder.CreateFPToSI(FloatVal, Builder.getIntNTy(32)); + Value *A0 = Builder.CreateFPToSI(FloatVal, Builder.getInt32Ty()); A1 = Builder.CreateSExt(A0, IntTy); } FPToI->replaceAllUsesWith(A1); @@ -425,8 +425,8 @@ static void expandIToFP(Instruction *IToFP) { AAddr0->addIncoming(IsSigned ? Sub : IntVal, IfThen4); AAddr0->addIncoming(Shl, SwBB); Value *A0 = Builder.CreateTrunc(AAddr0, Builder.getInt32Ty()); - Value *A1 = Builder.CreateLShr(A0, Builder.getIntN(32, 2)); - Value *A2 = Builder.CreateAnd(A1, Builder.getIntN(32, 1)); + Value *A1 = Builder.CreateLShr(A0, Builder.getInt32(2)); + Value *A2 = Builder.CreateAnd(A1, Builder.getInt32(1)); Value *Conv16 = Builder.CreateZExt(A2, IntTy); Value *Or17 = Builder.CreateOr(AAddr0, Conv16); Value *Inc = Builder.CreateAdd(Or17, Builder.getIntN(BitWidth, 1)); @@ -457,9 +457,9 @@ static void expandIToFP(Instruction *IToFP) { Value *Extract = Builder.CreateLShr(Shr21, Builder.getIntN(BitWidth, 32)); Value *ExtractT62 = nullptr; if (FloatWidth > 80) - ExtractT62 = Builder.CreateTrunc(Sub1, Builder.getIntNTy(64)); + ExtractT62 = Builder.CreateTrunc(Sub1, Builder.getInt64Ty()); else - ExtractT62 = Builder.CreateTrunc(Extract, Builder.getIntNTy(32)); + ExtractT62 = Builder.CreateTrunc(Extract, Builder.getInt32Ty()); Builder.CreateBr(IfEnd26); // if.else: @@ -475,7 +475,7 @@ static void expandIToFP(Instruction *IToFP) { Value *Extract65 = Builder.CreateLShr(Shl26, Builder.getIntN(BitWidth, 32)); Value *ExtractT66 = nullptr; if (FloatWidth > 80) - ExtractT66 = Builder.CreateTrunc(Sub2, Builder.getIntNTy(64)); + ExtractT66 = Builder.CreateTrunc(Sub2, Builder.getInt64Ty()); else ExtractT66 = Builder.CreateTrunc(Extract65, Builder.getInt32Ty()); Builder.CreateBr(IfEnd26); @@ -507,30 +507,29 @@ static void expandIToFP(Instruction *IToFP) { Builder.getIntN(BitWidth, 63)); And29 = Builder.CreateAnd(Shr, Temp2, "and29"); } else { - Value *Conv28 = Builder.CreateTrunc(Shr, Builder.getIntNTy(32)); + Value *Conv28 = Builder.CreateTrunc(Shr, Builder.getInt32Ty()); And29 = Builder.CreateAnd( - Conv28, ConstantInt::getSigned(Builder.getIntNTy(32), 0x80000000)); + Conv28, ConstantInt::getSigned(Builder.getInt32Ty(), 0x80000000)); } unsigned TempMod = FPMantissaWidth % 32; Value *And34 = nullptr; Value *Shl30 = nullptr; if (FloatWidth > 80) { TempMod += 32; - Value *Add = Builder.CreateShl(AAddr1Off32, Builder.getIntN(64, TempMod)); + Value *Add = Builder.CreateShl(AAddr1Off32, Builder.getInt64(TempMod)); Shl30 = Builder.CreateAdd( - Add, - Builder.getIntN(64, ((1ull << (62ull - TempMod)) - 1ull) << TempMod)); - And34 = Builder.CreateZExt(Shl30, Builder.getIntNTy(128)); + Add, Builder.getInt64(((1ull << (62ull - TempMod)) - 1ull) << TempMod)); + And34 = Builder.CreateZExt(Shl30, Builder.getInt128Ty()); } else { - Value *Add = Builder.CreateShl(E0, Builder.getIntN(32, TempMod)); + Value *Add = Builder.CreateShl(E0, Builder.getInt32(TempMod)); Shl30 = Builder.CreateAdd( - Add, Builder.getIntN(32, ((1 << (30 - TempMod)) - 1) << TempMod)); + Add, Builder.getInt32(((1 << (30 - TempMod)) - 1) << TempMod)); And34 = Builder.CreateAnd(FloatWidth > 32 ? AAddr1Off32 : AAddr1Off0, - Builder.getIntN(32, (1 << TempMod) - 1)); + Builder.getInt32((1 << TempMod) - 1)); } Value *Or35 = nullptr; if (FloatWidth > 80) { - Value *And29Trunc = Builder.CreateTrunc(And29, Builder.getIntNTy(128)); + Value *And29Trunc = Builder.CreateTrunc(And29, Builder.getInt128Ty()); Value *Or31 = Builder.CreateOr(And29Trunc, And34); Value *Or34 = Builder.CreateShl(Or31, Builder.getIntN(128, 64)); Value *Temp3 = Builder.CreateShl(Builder.getIntN(128, 1), diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp index df162fc..5e50898 100644 --- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp +++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp @@ -253,6 +253,21 @@ static bool isReInterleaveMask(ShuffleVectorInst *SVI, unsigned &Factor, return false; } +static Value *getMaskOperand(IntrinsicInst *II) { + switch (II->getIntrinsicID()) { + default: + llvm_unreachable("Unexpected intrinsic"); + case Intrinsic::vp_load: + return II->getOperand(1); + case Intrinsic::masked_load: + return II->getOperand(2); + case Intrinsic::vp_store: + return II->getOperand(2); + case Intrinsic::masked_store: + return II->getOperand(3); + } +} + // Return the corresponded deinterleaved mask, or nullptr if there is no valid // mask. static Value *getMask(Value *WideMask, unsigned Factor, @@ -268,17 +283,13 @@ bool InterleavedAccessImpl::lowerInterleavedLoad( if (isa<ScalableVectorType>(Load->getType())) return false; - if (auto *LI = dyn_cast<LoadInst>(Load)) { - if (!LI->isSimple()) - return false; - } else if (auto *VPLoad = dyn_cast<VPIntrinsic>(Load)) { - assert(VPLoad->getIntrinsicID() == Intrinsic::vp_load); - // Require a constant mask. - if (!isa<ConstantVector>(VPLoad->getMaskParam())) - return false; - } else { - llvm_unreachable("unsupported load operation"); - } + auto *LI = dyn_cast<LoadInst>(Load); + auto *II = dyn_cast<IntrinsicInst>(Load); + if (!LI && !II) + return false; + + if (LI && !LI->isSimple()) + return false; // Check if all users of this load are shufflevectors. If we encounter any // users that are extractelement instructions or binary operators, we save @@ -330,7 +341,7 @@ bool InterleavedAccessImpl::lowerInterleavedLoad( // Holds the corresponding index for each DE-interleave shuffle. SmallVector<unsigned, 4> Indices; - Type *VecTy = FirstSVI->getType(); + VectorType *VecTy = cast<VectorType>(FirstSVI->getType()); // Check if other shufflevectors are also DE-interleaved of the same type // and factor as the first shufflevector. @@ -368,13 +379,16 @@ bool InterleavedAccessImpl::lowerInterleavedLoad( replaceBinOpShuffles(BinOpShuffles.getArrayRef(), Shuffles, Load); Value *Mask = nullptr; - if (auto *VPLoad = dyn_cast<VPIntrinsic>(Load)) { - Mask = getMask(VPLoad->getMaskParam(), Factor, cast<VectorType>(VecTy)); + if (LI) { + LLVM_DEBUG(dbgs() << "IA: Found an interleaved load: " << *Load << "\n"); + } else { + // Check mask operand. Handle both all-true/false and interleaved mask. + Mask = getMask(getMaskOperand(II), Factor, VecTy); if (!Mask) return false; - LLVM_DEBUG(dbgs() << "IA: Found an interleaved vp.load: " << *Load << "\n"); - } else { - LLVM_DEBUG(dbgs() << "IA: Found an interleaved load: " << *Load << "\n"); + + LLVM_DEBUG(dbgs() << "IA: Found an interleaved vp.load or masked.load: " + << *Load << "\n"); } // Try to create target specific intrinsics to replace the load and @@ -491,18 +505,16 @@ bool InterleavedAccessImpl::tryReplaceExtracts( bool InterleavedAccessImpl::lowerInterleavedStore( Instruction *Store, SmallSetVector<Instruction *, 32> &DeadInsts) { Value *StoredValue; - if (auto *SI = dyn_cast<StoreInst>(Store)) { + auto *SI = dyn_cast<StoreInst>(Store); + auto *II = dyn_cast<IntrinsicInst>(Store); + if (SI) { if (!SI->isSimple()) return false; StoredValue = SI->getValueOperand(); - } else if (auto *VPStore = dyn_cast<VPIntrinsic>(Store)) { - assert(VPStore->getIntrinsicID() == Intrinsic::vp_store); - // Require a constant mask. - if (!isa<ConstantVector>(VPStore->getMaskParam())) - return false; - StoredValue = VPStore->getArgOperand(0); } else { - llvm_unreachable("unsupported store operation"); + assert(II->getIntrinsicID() == Intrinsic::vp_store || + II->getIntrinsicID() == Intrinsic::masked_store); + StoredValue = II->getArgOperand(0); } auto *SVI = dyn_cast<ShuffleVectorInst>(StoredValue); @@ -518,46 +530,26 @@ bool InterleavedAccessImpl::lowerInterleavedStore( assert(NumStoredElements % Factor == 0 && "number of stored element should be a multiple of Factor"); - if (auto *VPStore = dyn_cast<VPIntrinsic>(Store)) { + Value *Mask = nullptr; + if (SI) { + LLVM_DEBUG(dbgs() << "IA: Found an interleaved store: " << *Store << "\n"); + } else { + // Check mask operand. Handle both all-true/false and interleaved mask. unsigned LaneMaskLen = NumStoredElements / Factor; - Value *LaneMask = getMask(VPStore->getMaskParam(), Factor, - ElementCount::getFixed(LaneMaskLen)); - if (!LaneMask) + Mask = getMask(getMaskOperand(II), Factor, + ElementCount::getFixed(LaneMaskLen)); + if (!Mask) return false; - LLVM_DEBUG(dbgs() << "IA: Found an interleaved vp.store: " << *Store - << "\n"); - - IRBuilder<> Builder(VPStore); - // We need to effectively de-interleave the shufflemask - // because lowerInterleavedVPStore expects individual de-interleaved - // values. - SmallVector<Value *, 10> NewShuffles; - SmallVector<int, 16> NewShuffleMask(LaneMaskLen); - auto ShuffleMask = SVI->getShuffleMask(); - - for (unsigned i = 0; i < Factor; i++) { - for (unsigned j = 0; j < LaneMaskLen; j++) - NewShuffleMask[j] = ShuffleMask[i + Factor * j]; - - NewShuffles.push_back(Builder.CreateShuffleVector( - SVI->getOperand(0), SVI->getOperand(1), NewShuffleMask)); - } - - // Try to create target specific intrinsics to replace the vp.store and - // shuffle. - if (!TLI->lowerInterleavedVPStore(VPStore, LaneMask, NewShuffles)) - // We already created new shuffles. - return true; - } else { - LLVM_DEBUG(dbgs() << "IA: Found an interleaved store: " << *Store << "\n"); - - // Try to create target specific intrinsics to replace the store and - // shuffle. - if (!TLI->lowerInterleavedStore(cast<StoreInst>(Store), SVI, Factor)) - return false; + LLVM_DEBUG(dbgs() << "IA: Found an interleaved vp.store or masked.store: " + << *Store << "\n"); } + // Try to create target specific intrinsics to replace the store and + // shuffle. + if (!TLI->lowerInterleavedStore(Store, Mask, SVI, Factor)) + return false; + // Already have a new target specific interleaved store. Erase the old store. DeadInsts.insert(Store); DeadInsts.insert(SVI); @@ -595,6 +587,27 @@ static Value *getMask(Value *WideMask, unsigned Factor, } } + if (auto *SVI = dyn_cast<ShuffleVectorInst>(WideMask)) { + // Check that the shuffle mask is: a) an interleave, b) all of the same + // set of the elements, and c) contained by the first source. (c) could + // be relaxed if desired. + unsigned NumSrcElts = + cast<FixedVectorType>(SVI->getOperand(1)->getType())->getNumElements(); + SmallVector<unsigned> StartIndexes; + if (ShuffleVectorInst::isInterleaveMask(SVI->getShuffleMask(), Factor, + NumSrcElts * 2, StartIndexes) && + llvm::all_of(StartIndexes, [](unsigned Start) { return Start == 0; }) && + llvm::all_of(SVI->getShuffleMask(), [&NumSrcElts](int Idx) { + return Idx < (int)NumSrcElts; + })) { + auto *LeafMaskTy = + VectorType::get(Type::getInt1Ty(SVI->getContext()), LeafValueEC); + IRBuilder<> Builder(SVI); + return Builder.CreateExtractVector(LeafMaskTy, SVI->getOperand(0), + uint64_t(0)); + } + } + return nullptr; } @@ -621,21 +634,12 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic( << " and factor = " << Factor << "\n"); } else { assert(II); - - // Check mask operand. Handle both all-true/false and interleaved mask. - Value *WideMask; - switch (II->getIntrinsicID()) { - default: + if (II->getIntrinsicID() != Intrinsic::masked_load && + II->getIntrinsicID() != Intrinsic::vp_load) return false; - case Intrinsic::vp_load: - WideMask = II->getOperand(1); - break; - case Intrinsic::masked_load: - WideMask = II->getOperand(2); - break; - } - Mask = getMask(WideMask, Factor, getDeinterleavedVectorType(DI)); + // Check mask operand. Handle both all-true/false and interleaved mask. + Mask = getMask(getMaskOperand(II), Factor, getDeinterleavedVectorType(DI)); if (!Mask) return false; @@ -672,19 +676,11 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic( Value *Mask = nullptr; if (II) { - // Check mask operand. Handle both all-true/false and interleaved mask. - Value *WideMask; - switch (II->getIntrinsicID()) { - default: + if (II->getIntrinsicID() != Intrinsic::masked_store && + II->getIntrinsicID() != Intrinsic::vp_store) return false; - case Intrinsic::vp_store: - WideMask = II->getOperand(2); - break; - case Intrinsic::masked_store: - WideMask = II->getOperand(3); - break; - } - Mask = getMask(WideMask, Factor, + // Check mask operand. Handle both all-true/false and interleaved mask. + Mask = getMask(getMaskOperand(II), Factor, cast<VectorType>(InterleaveValues[0]->getType())); if (!Mask) return false; @@ -718,11 +714,13 @@ bool InterleavedAccessImpl::runOnFunction(Function &F) { using namespace PatternMatch; for (auto &I : instructions(F)) { if (match(&I, m_CombineOr(m_Load(m_Value()), - m_Intrinsic<Intrinsic::vp_load>()))) + m_Intrinsic<Intrinsic::vp_load>())) || + match(&I, m_Intrinsic<Intrinsic::masked_load>())) Changed |= lowerInterleavedLoad(&I, DeadInsts); if (match(&I, m_CombineOr(m_Store(m_Value(), m_Value()), - m_Intrinsic<Intrinsic::vp_store>()))) + m_Intrinsic<Intrinsic::vp_store>())) || + match(&I, m_Intrinsic<Intrinsic::masked_store>())) Changed |= lowerInterleavedStore(&I, DeadInsts); if (auto *II = dyn_cast<IntrinsicInst>(&I)) { diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp index 7710b50..bc4e299 100644 --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -815,6 +815,9 @@ static void printMI(raw_ostream &OS, MFPrintState &State, if (MI.getFlag(MachineInstr::SameSign)) OS << "samesign "; + // NOTE: Please add new MIFlags also to the MI_FLAGS_STR in + // llvm/utils/update_mir_test_checks.py. + OS << TII->getName(MI.getOpcode()); LS = ListSeparator(); diff --git a/llvm/lib/CodeGen/MachineInstrBundle.cpp b/llvm/lib/CodeGen/MachineInstrBundle.cpp index 34896c6..4da0184 100644 --- a/llvm/lib/CodeGen/MachineInstrBundle.cpp +++ b/llvm/lib/CodeGen/MachineInstrBundle.cpp @@ -83,27 +83,6 @@ llvm::createUnpackMachineBundles( return new UnpackMachineBundles(std::move(Ftor)); } -namespace { - class FinalizeMachineBundles : public MachineFunctionPass { - public: - static char ID; // Pass identification - FinalizeMachineBundles() : MachineFunctionPass(ID) { - initializeFinalizeMachineBundlesPass(*PassRegistry::getPassRegistry()); - } - - bool runOnMachineFunction(MachineFunction &MF) override; - }; -} // end anonymous namespace - -char FinalizeMachineBundles::ID = 0; -char &llvm::FinalizeMachineBundlesID = FinalizeMachineBundles::ID; -INITIALIZE_PASS(FinalizeMachineBundles, "finalize-mi-bundles", - "Finalize machine instruction bundles", false, false) - -bool FinalizeMachineBundles::runOnMachineFunction(MachineFunction &MF) { - return llvm::finalizeBundles(MF); -} - /// Return the first found DebugLoc that has a DILocation, given a range of /// instructions. The search range is from FirstMI to LastMI (exclusive). If no /// DILocation is found, then an empty location is returned. @@ -359,3 +338,13 @@ PhysRegInfo llvm::AnalyzePhysRegInBundle(const MachineInstr &MI, Register Reg, return PRI; } + +PreservedAnalyses +llvm::FinalizeBundleTestPass::run(MachineFunction &MF, + MachineFunctionAnalysisManager &) { + // For testing purposes, bundle the entire contents of each basic block + // except for terminators. + for (MachineBasicBlock &MBB : MF) + finalizeBundle(MBB, MBB.instr_begin(), MBB.getFirstInstrTerminator()); + return PreservedAnalyses::none(); +} diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp index e144111..286fbfd 100644 --- a/llvm/lib/CodeGen/MachineLICM.cpp +++ b/llvm/lib/CodeGen/MachineLICM.cpp @@ -49,7 +49,6 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <cassert> #include <limits> #include <vector> diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index f12b18d..d3df434 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -12375,11 +12375,8 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))) { // Any flags available in a select/setcc fold will be on the setcc as they // migrated from fcmp - Flags = N0->getFlags(); - SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, - N2, N0.getOperand(2)); - SelectNode->setFlags(Flags); - return SelectNode; + return DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, N2, + N0.getOperand(2), N0->getFlags()); } if (SDValue ABD = foldSelectToABD(Cond0, Cond1, N1, N2, CC, DL)) @@ -16738,7 +16735,8 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) { // Fold freeze(op(x, ...)) -> op(freeze(x), ...). // Try to push freeze through instructions that propagate but don't produce // poison as far as possible. If an operand of freeze follows three - // conditions 1) one-use, and 2) does not produce poison then push + // conditions 1) one-use, 2) does not produce poison, and 3) has all but one + // guaranteed-non-poison operands (or is a BUILD_VECTOR or similar) then push // the freeze through to the operands that are not guaranteed non-poison. // NOTE: we will strip poison-generating flags, so ignore them here. if (DAG.canCreateUndefOrPoison(N0, /*PoisonOnly*/ false, @@ -16746,6 +16744,18 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) { N0->getNumValues() != 1 || !N0->hasOneUse()) return SDValue(); + // TOOD: we should always allow multiple operands, however this increases the + // likelihood of infinite loops due to the ReplaceAllUsesOfValueWith call + // below causing later nodes that share frozen operands to fold again and no + // longer being able to confirm other operands are not poison due to recursion + // depth limits on isGuaranteedNotToBeUndefOrPoison. + bool AllowMultipleMaybePoisonOperands = + N0.getOpcode() == ISD::SELECT_CC || N0.getOpcode() == ISD::SETCC || + N0.getOpcode() == ISD::BUILD_VECTOR || + N0.getOpcode() == ISD::BUILD_PAIR || + N0.getOpcode() == ISD::VECTOR_SHUFFLE || + N0.getOpcode() == ISD::CONCAT_VECTORS || N0.getOpcode() == ISD::FMUL; + // Avoid turning a BUILD_VECTOR that can be recognized as "all zeros", "all // ones" or "constant" into something that depends on FrozenUndef. We can // instead pick undef values to keep those properties, while at the same time @@ -16772,8 +16782,16 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) { if (DAG.isGuaranteedNotToBeUndefOrPoison(Op, /*PoisonOnly*/ false, /*Depth*/ 1)) continue; - if (MaybePoisonOperands.insert(Op).second) + bool HadMaybePoisonOperands = !MaybePoisonOperands.empty(); + bool IsNewMaybePoisonOperand = MaybePoisonOperands.insert(Op).second; + if (IsNewMaybePoisonOperand) MaybePoisonOperandNumbers.push_back(OpNo); + if (!HadMaybePoisonOperands) + continue; + if (IsNewMaybePoisonOperand && !AllowMultipleMaybePoisonOperands) { + // Multiple maybe-poison ops when not allowed - bail out. + return SDValue(); + } } // NOTE: the whole op may be not guaranteed to not be undef or poison because // it could create undef or poison due to it's poison-generating flags. diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 85efb1b..8c8daef 100644 --- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -402,7 +402,12 @@ void InstrEmitter::AddOperand(MachineInstrBuilder &MIB, SDValue Op, AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap, IsDebug, IsClone, IsCloned); } else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { - MIB.addImm(C->getSExtValue()); + if (C->getAPIntValue().getSignificantBits() <= 64) { + MIB.addImm(C->getSExtValue()); + } else { + MIB.addCImm( + ConstantInt::get(MF->getFunction().getContext(), C->getAPIntValue())); + } } else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Op)) { MIB.addFPImm(F->getConstantFPValue()); } else if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Op)) { diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 7266940..74172b2 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -2785,19 +2785,17 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node, // In strict mode, we must avoid spurious exceptions, and therefore // must make sure to only emit a single STRICT_SINT_TO_FP. SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0); - Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, - { Node->getOperand(0), InCvt }); - Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, - { Fast.getValue(1), Fast, Fast }); - Chain = Slow.getValue(1); // The STRICT_SINT_TO_FP inherits the exception mode from the // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can // never raise any exception. SDNodeFlags Flags; Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept()); - Fast->setFlags(Flags); + Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {DestVT, MVT::Other}, + {Node->getOperand(0), InCvt}, Flags); Flags.setNoFPExcept(true); - Slow->setFlags(Flags); + Slow = DAG.getNode(ISD::STRICT_FADD, dl, {DestVT, MVT::Other}, + {Fast.getValue(1), Fast, Fast}, Flags); + Chain = Slow.getValue(1); } else { SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or); Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt); @@ -3407,14 +3405,12 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { EVT VT = Operand.getValueType(); SDValue One = DAG.getConstantFP(1.0, dl, VT); SDValue Chain = DAG.getEntryNode(); - SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, dl, {VT, MVT::Other}, - {Chain, Operand, One}); - // Propagate existing flags on canonicalize, and additionally set // NoFPExcept. SDNodeFlags CanonicalizeFlags = Node->getFlags(); CanonicalizeFlags.setNoFPExcept(true); - Mul->setFlags(CanonicalizeFlags); + SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, dl, {VT, MVT::Other}, + {Chain, Operand, One}, CanonicalizeFlags); Results.push_back(Mul); break; @@ -4150,15 +4146,14 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { Tmp2 = Node->getOperand(1); Tmp3 = Node->getOperand(2); if (Tmp1.getOpcode() == ISD::SETCC) { - Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), - Tmp2, Tmp3, - cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); + Tmp1 = DAG.getSelectCC( + dl, Tmp1.getOperand(0), Tmp1.getOperand(1), Tmp2, Tmp3, + cast<CondCodeSDNode>(Tmp1.getOperand(2))->get(), Node->getFlags()); } else { - Tmp1 = DAG.getSelectCC(dl, Tmp1, - DAG.getConstant(0, dl, Tmp1.getValueType()), - Tmp2, Tmp3, ISD::SETNE); + Tmp1 = + DAG.getSelectCC(dl, Tmp1, DAG.getConstant(0, dl, Tmp1.getValueType()), + Tmp2, Tmp3, ISD::SETNE, Node->getFlags()); } - Tmp1->setFlags(Node->getFlags()); Results.push_back(Tmp1); break; case ISD::BR_JT: { @@ -4296,8 +4291,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { EVT Tmp1VT = Tmp1.getValueType(); Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, DAG.getBoolConstant(true, dl, VT, Tmp1VT), - DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3); - Tmp1->setFlags(Node->getFlags()); + DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3, + Node->getFlags()); Results.push_back(Tmp1); break; } @@ -4335,8 +4330,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) { // Use the new condition code and swap true and false Legalized = true; - Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); - Tmp1->setFlags(Node->getFlags()); + Tmp1 = + DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC, Node->getFlags()); } else { // If The inverse is not legal, then try to swap the arguments using // the inverse condition code. @@ -4345,8 +4340,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { // The swapped inverse condition is legal, so swap true and false, // lhs and rhs. Legalized = true; - Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); - Tmp1->setFlags(Node->getFlags()); + Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC, + Node->getFlags()); } } @@ -4365,15 +4360,14 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { // If we expanded the SETCC by swapping LHS and RHS, or by inverting the // condition code, create a new SELECT_CC node. if (CC.getNode()) { - Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), - Tmp1, Tmp2, Tmp3, Tmp4, CC); + Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, + Tmp2, Tmp3, Tmp4, CC, Node->getFlags()); } else { Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType()); CC = DAG.getCondCode(ISD::SETNE); Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, - Tmp2, Tmp3, Tmp4, CC); + Tmp2, Tmp3, Tmp4, CC, Node->getFlags()); } - Tmp1->setFlags(Node->getFlags()); } Results.push_back(Tmp1); break; diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index e5704c0..583a85a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -22,6 +22,7 @@ #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/DiagnosticInfo.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/KnownBits.h" #include "llvm/Support/raw_ostream.h" @@ -357,6 +358,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { case ISD::PATCHPOINT: Res = PromoteIntRes_PATCHPOINT(N); break; + case ISD::READ_REGISTER: + Res = PromoteIntRes_READ_REGISTER(N); + break; } // If the result is null then the sub-method took care of registering it. @@ -2076,6 +2080,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { case ISD::PATCHPOINT: Res = PromoteIntOp_PATCHPOINT(N, OpNo); break; + case ISD::WRITE_REGISTER: + Res = PromoteIntOp_WRITE_REGISTER(N, OpNo); + break; case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: case ISD::EXPERIMENTAL_VP_STRIDED_STORE: Res = PromoteIntOp_VP_STRIDED(N, OpNo); @@ -2853,6 +2860,15 @@ SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); } +SDValue DAGTypeLegalizer::PromoteIntOp_WRITE_REGISTER(SDNode *N, + unsigned OpNo) { + const Function &Fn = DAG.getMachineFunction().getFunction(); + Fn.getContext().diagnose(DiagnosticInfoLegalizationFailure( + "cannot use llvm.write_register with illegal type", Fn, + N->getDebugLoc())); + return N->getOperand(0); +} + SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); @@ -3127,6 +3143,10 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { case ISD::VSCALE: ExpandIntRes_VSCALE(N, Lo, Hi); break; + + case ISD::READ_REGISTER: + ExpandIntRes_READ_REGISTER(N, Lo, Hi); + break; } // If Lo/Hi is null, the sub-method took care of registering results etc. @@ -5471,6 +5491,18 @@ void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo, SplitInteger(Res, Lo, Hi); } +void DAGTypeLegalizer::ExpandIntRes_READ_REGISTER(SDNode *N, SDValue &Lo, + SDValue &Hi) { + const Function &Fn = DAG.getMachineFunction().getFunction(); + Fn.getContext().diagnose(DiagnosticInfoLegalizationFailure( + "cannot use llvm.read_register with illegal type", Fn, N->getDebugLoc())); + ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); + EVT LoVT, HiVT; + std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); + Lo = DAG.getPOISON(LoVT); + Hi = DAG.getPOISON(HiVT); +} + //===----------------------------------------------------------------------===// // Integer Operand Expansion //===----------------------------------------------------------------------===// @@ -5537,6 +5569,9 @@ bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { case ISD::EXPERIMENTAL_VP_STRIDED_STORE: Res = ExpandIntOp_VP_STRIDED(N, OpNo); break; + case ISD::WRITE_REGISTER: + Res = ExpandIntOp_WRITE_REGISTER(N, OpNo); + break; } // If the result is null, the sub-method took care of registering results etc. @@ -5935,6 +5970,15 @@ SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); } +SDValue DAGTypeLegalizer::ExpandIntOp_WRITE_REGISTER(SDNode *N, unsigned OpNo) { + const Function &Fn = DAG.getMachineFunction().getFunction(); + Fn.getContext().diagnose(DiagnosticInfoLegalizationFailure( + "cannot use llvm.write_register with illegal type", Fn, + N->getDebugLoc())); + + return N->getOperand(0); +} + SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) { SDLoc dl(N); @@ -6332,6 +6376,16 @@ SDValue DAGTypeLegalizer::PromoteIntRes_PATCHPOINT(SDNode *N) { return Res.getValue(0); } +SDValue DAGTypeLegalizer::PromoteIntRes_READ_REGISTER(SDNode *N) { + const Function &Fn = DAG.getMachineFunction().getFunction(); + Fn.getContext().diagnose(DiagnosticInfoLegalizationFailure( + "cannot use llvm.read_register with illegal type", Fn, N->getDebugLoc())); + + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); + return DAG.getPOISON(NVT); +} + SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { SDLoc dl(N); SDValue V0 = GetPromotedInteger(N->getOperand(0)); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 9b53724..2e13b18 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -378,6 +378,7 @@ private: SDValue PromoteIntRes_VPFunnelShift(SDNode *N); SDValue PromoteIntRes_IS_FPCLASS(SDNode *N); SDValue PromoteIntRes_PATCHPOINT(SDNode *N); + SDValue PromoteIntRes_READ_REGISTER(SDNode *N); SDValue PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N); SDValue PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N); SDValue PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N); @@ -428,6 +429,7 @@ private: SDValue PromoteIntOp_SET_ROUNDING(SDNode *N); SDValue PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo); + SDValue PromoteIntOp_WRITE_REGISTER(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_VECTOR_HISTOGRAM(SDNode *N, unsigned OpNo); @@ -511,6 +513,7 @@ private: void ExpandIntRes_FunnelShift (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_VSCALE (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandIntRes_READ_REGISTER(SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandShiftByConstant(SDNode *N, const APInt &Amt, SDValue &Lo, SDValue &Hi); @@ -534,6 +537,7 @@ private: SDValue ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo); SDValue ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo); SDValue ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo); + SDValue ExpandIntOp_WRITE_REGISTER(SDNode *N, unsigned OpNo); void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &dl); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index f908a66..d2ecc133 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -2087,11 +2087,10 @@ void VectorLegalizer::ExpandSETCC(SDNode *Node, // Otherwise, SETCC for the given comparison type must be completely // illegal; expand it into a SELECT_CC. EVT VT = Node->getValueType(0); - LHS = - DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS, - DAG.getBoolConstant(true, dl, VT, LHS.getValueType()), - DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), CC); - LHS->setFlags(Node->getFlags()); + LHS = DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS, + DAG.getBoolConstant(true, dl, VT, LHS.getValueType()), + DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), + CC, Node->getFlags()); } Results.push_back(LHS); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 32c5961..1661814 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -372,9 +372,9 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N, SDVTList ScalarVTs = DAG.getVTList( ResVT.getVectorElementType(), OvVT.getVectorElementType()); - SDNode *ScalarNode = DAG.getNode( - N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode(); - ScalarNode->setFlags(N->getFlags()); + SDNode *ScalarNode = DAG.getNode(N->getOpcode(), DL, ScalarVTs, + {ScalarLHS, ScalarRHS}, N->getFlags()) + .getNode(); // Replace the other vector result not being explicitly scalarized here. unsigned OtherNo = 1 - ResNo; @@ -1898,7 +1898,7 @@ SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) { NE = ResNE; //The results of each unrolled operation, including the chain. - EVT ChainVTs[] = {EltVT, MVT::Other}; + SDVTList ChainVTs = DAG.getVTList(EltVT, MVT::Other); SmallVector<SDValue, 8> Chains; unsigned i; @@ -1914,8 +1914,8 @@ SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) { Operands[j] = Operand; } } - SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands); - Scalar.getNode()->setFlags(N->getFlags()); + SDValue Scalar = + DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands, N->getFlags()); //Add in the scalar as well as its chain value to the //result vectors. @@ -1956,10 +1956,10 @@ void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo, unsigned Opcode = N->getOpcode(); SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT); SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT); - SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode(); - SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode(); - LoNode->setFlags(N->getFlags()); - HiNode->setFlags(N->getFlags()); + SDNode *LoNode = + DAG.getNode(Opcode, dl, LoVTs, {LoLHS, LoRHS}, N->getFlags()).getNode(); + SDNode *HiNode = + DAG.getNode(Opcode, dl, HiVTs, {HiLHS, HiRHS}, N->getFlags()).getNode(); Lo = SDValue(LoNode, ResNo); Hi = SDValue(HiNode, ResNo); @@ -2669,10 +2669,8 @@ void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N, else std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); - Lo = DAG.getNode(N->getOpcode(), dl, {LoVT, LoVT1}, Lo); - Hi = DAG.getNode(N->getOpcode(), dl, {HiVT, HiVT1}, Hi); - Lo->setFlags(N->getFlags()); - Hi->setFlags(N->getFlags()); + Lo = DAG.getNode(N->getOpcode(), dl, {LoVT, LoVT1}, Lo, N->getFlags()); + Hi = DAG.getNode(N->getOpcode(), dl, {HiVT, HiVT1}, Hi, N->getFlags()); SDNode *HiNode = Hi.getNode(); SDNode *LoNode = Lo.getNode(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index e1f2e95..773ff48 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -10559,7 +10559,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef<SDUse> Ops) { switch (Ops.size()) { case 0: return getNode(Opcode, DL, VT); - case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0])); + case 1: return getNode(Opcode, DL, VT, Ops[0].get()); case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); default: break; @@ -10695,7 +10695,16 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) { - return getNode(Opcode, DL, getVTList(ResultTys), Ops); + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNode(Opcode, DL, getVTList(ResultTys), Ops, Flags); +} + +SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, + ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops, + const SDNodeFlags Flags) { + return getNode(Opcode, DL, getVTList(ResultTys), Ops, Flags); } SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, @@ -10851,26 +10860,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) && "Invalid STRICT_FP_ROUND!"); break; -#if 0 - // FIXME: figure out how to safely handle things like - // int foo(int x) { return 1 << (x & 255); } - // int bar() { return foo(256); } - case ISD::SRA_PARTS: - case ISD::SRL_PARTS: - case ISD::SHL_PARTS: - if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && - cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) - return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); - else if (N3.getOpcode() == ISD::AND) - if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { - // If the and is only masking out bits that cannot effect the shift, - // eliminate the and. - unsigned NumBits = VT.getScalarSizeInBits()*2; - if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) - return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); - } - break; -#endif } // Memoize the node unless it returns a glue result. diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 94d1b3b..1764910 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -8634,9 +8634,8 @@ TargetLowering::createSelectForFMINNUM_FMAXNUM(SDNode *Node, return SDValue(); SDValue Op1 = Node->getOperand(0); SDValue Op2 = Node->getOperand(1); - SDValue SelCC = DAG.getSelectCC(SDLoc(Node), Op1, Op2, Op1, Op2, Pred); - SelCC->setFlags(Node->getFlags()); - return SelCC; + return DAG.getSelectCC(SDLoc(Node), Op1, Op2, Op1, Op2, Pred, + Node->getFlags()); } return SDValue(); @@ -11995,8 +11994,7 @@ SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node, // Get the mask value and add it to the current output position. This // either increments by 1 if MaskI is true or adds 0 otherwise. // Freeze in case we have poison/undef mask entries. - SDValue MaskI = - DAG.getFreeze(DAG.getExtractVectorElt(DL, MaskScalarVT, Mask, I)); + SDValue MaskI = DAG.getExtractVectorElt(DL, MaskScalarVT, Mask, I); MaskI = DAG.getFreeze(MaskI); MaskI = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, MaskI); MaskI = DAG.getNode(ISD::ZERO_EXTEND, DL, PositionVT, MaskI); diff --git a/llvm/lib/CodeGen/StackProtector.cpp b/llvm/lib/CodeGen/StackProtector.cpp index b79911b..2a8234a 100644 --- a/llvm/lib/CodeGen/StackProtector.cpp +++ b/llvm/lib/CodeGen/StackProtector.cpp @@ -588,7 +588,14 @@ bool InsertStackProtectors(const TargetMachine *TM, Function *F, continue; Instruction *CheckLoc = dyn_cast<ReturnInst>(BB.getTerminator()); if (!CheckLoc && !DisableCheckNoReturn) - for (auto &Inst : BB) + for (auto &Inst : BB) { + if (IntrinsicInst *IB = dyn_cast<IntrinsicInst>(&Inst); + IB && (IB->getIntrinsicID() == Intrinsic::eh_sjlj_callsite)) { + // eh_sjlj_callsite has to be in same BB as the + // bb terminator. Don't insert within this range. + CheckLoc = IB; + break; + } if (auto *CB = dyn_cast<CallBase>(&Inst)) // Do stack check before noreturn calls that aren't nounwind (e.g: // __cxa_throw). @@ -596,6 +603,7 @@ bool InsertStackProtectors(const TargetMachine *TM, Function *F, CheckLoc = CB; break; } + } if (!CheckLoc) continue; diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index 7e501a9..408d07b 100644 --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -42,7 +42,6 @@ #include "llvm/IR/Mangler.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" -#include "llvm/IR/PseudoProbe.h" #include "llvm/IR/Type.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCAsmInfoDarwin.h" @@ -996,7 +995,7 @@ MCSection *TargetLoweringObjectFileELF::getSectionForLSDA( if (!LSDASection || (!F.hasComdat() && !TM.getFunctionSections())) return LSDASection; - const auto *LSDA = cast<MCSectionELF>(LSDASection); + const auto *LSDA = static_cast<const MCSectionELF *>(LSDASection); unsigned Flags = LSDA->getFlags(); const MCSymbolELF *LinkedToSym = nullptr; StringRef Group; @@ -2055,14 +2054,14 @@ MCSection *TargetLoweringObjectFileCOFF::getStaticCtorSection( unsigned Priority, const MCSymbol *KeySym) const { return getCOFFStaticStructorSection( getContext(), getContext().getTargetTriple(), true, Priority, KeySym, - cast<MCSectionCOFF>(StaticCtorSection)); + static_cast<MCSectionCOFF *>(StaticCtorSection)); } MCSection *TargetLoweringObjectFileCOFF::getStaticDtorSection( unsigned Priority, const MCSymbol *KeySym) const { return getCOFFStaticStructorSection( getContext(), getContext().getTargetTriple(), false, Priority, KeySym, - cast<MCSectionCOFF>(StaticDtorSection)); + static_cast<MCSectionCOFF *>(StaticDtorSection)); } const MCExpr *TargetLoweringObjectFileCOFF::lowerRelativeReference( @@ -2389,23 +2388,25 @@ TargetLoweringObjectFileXCOFF::getTargetSymbol(const GlobalValue *GV, // here. if (const GlobalObject *GO = dyn_cast<GlobalObject>(GV)) { if (GO->isDeclarationForLinker()) - return cast<MCSectionXCOFF>(getSectionForExternalReference(GO, TM)) + return static_cast<const MCSectionXCOFF *>( + getSectionForExternalReference(GO, TM)) ->getQualNameSymbol(); if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV)) if (GVar->hasAttribute("toc-data")) - return cast<MCSectionXCOFF>( + return static_cast<const MCSectionXCOFF *>( SectionForGlobal(GVar, SectionKind::getData(), TM)) ->getQualNameSymbol(); SectionKind GOKind = getKindForGlobal(GO, TM); if (GOKind.isText()) - return cast<MCSectionXCOFF>( + return static_cast<const MCSectionXCOFF *>( getSectionForFunctionDescriptor(cast<Function>(GO), TM)) ->getQualNameSymbol(); if ((TM.getDataSections() && !GO->hasSection()) || GO->hasCommonLinkage() || GOKind.isBSSLocal() || GOKind.isThreadBSSLocal()) - return cast<MCSectionXCOFF>(SectionForGlobal(GO, GOKind, TM)) + return static_cast<const MCSectionXCOFF *>( + SectionForGlobal(GO, GOKind, TM)) ->getQualNameSymbol(); } @@ -2741,7 +2742,7 @@ MCSection *TargetLoweringObjectFileXCOFF::getSectionForTOCEntry( MCSection *TargetLoweringObjectFileXCOFF::getSectionForLSDA( const Function &F, const MCSymbol &FnSym, const TargetMachine &TM) const { - auto *LSDA = cast<MCSectionXCOFF>(LSDASection); + auto *LSDA = static_cast<MCSectionXCOFF *>(LSDASection); if (TM.getFunctionSections()) { // If option -ffunction-sections is on, append the function name to the // name of the LSDA csect so that each function has its own LSDA csect. |