diff options
Diffstat (limited to 'llvm/lib/Transforms')
28 files changed, 350 insertions, 242 deletions
diff --git a/llvm/lib/Transforms/Coroutines/CoroCloner.h b/llvm/lib/Transforms/Coroutines/CoroCloner.h index 26ec4f3..e05fe28 100644 --- a/llvm/lib/Transforms/Coroutines/CoroCloner.h +++ b/llvm/lib/Transforms/Coroutines/CoroCloner.h @@ -1,3 +1,4 @@ +//===----------------------------------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -19,9 +20,7 @@ #include "llvm/Transforms/Coroutines/CoroInstr.h" #include "llvm/Transforms/Utils/ValueMapper.h" -namespace llvm { - -namespace coro { +namespace llvm::coro { enum class CloneKind { /// The shared resume function for a switch lowering. @@ -149,8 +148,6 @@ public: } }; -} // end namespace coro - -} // end namespace llvm +} // end namespace llvm::coro #endif // LLVM_LIB_TRANSFORMS_COROUTINES_COROCLONER_H diff --git a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp index 471b9eb..cdb5852 100644 --- a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp @@ -38,7 +38,7 @@ public: AnyResumeFnPtrTy(PointerType::getUnqual(Context)) {} void lowerEarlyIntrinsics(Function &F); }; -} +} // namespace // Replace a direct call to coro.resume or coro.destroy with an indirect call to // an address returned by coro.subfn.addr intrinsic. This is done so that diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h index 52f4ffe..cc47a55 100644 --- a/llvm/lib/Transforms/Coroutines/CoroInternal.h +++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h @@ -16,11 +16,7 @@ #include "llvm/Transforms/Coroutines/CoroInstr.h" #include "llvm/Transforms/Coroutines/CoroShape.h" -namespace llvm { - -class CallGraph; - -namespace coro { +namespace llvm::coro { bool isSuspendBlock(BasicBlock *BB); bool declaresAnyIntrinsic(const Module &M); @@ -61,7 +57,6 @@ void normalizeCoroutine(Function &F, coro::Shape &Shape, CallInst *createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, TargetTransformInfo &TTI, ArrayRef<Value *> Arguments, IRBuilder<> &); -} // End namespace coro. -} // End namespace llvm +} // End namespace llvm::coro #endif diff --git a/llvm/lib/Transforms/Coroutines/MaterializationUtils.cpp b/llvm/lib/Transforms/Coroutines/MaterializationUtils.cpp index 6aaabca..f2444da 100644 --- a/llvm/lib/Transforms/Coroutines/MaterializationUtils.cpp +++ b/llvm/lib/Transforms/Coroutines/MaterializationUtils.cpp @@ -137,8 +137,7 @@ struct RematGraph { } // namespace -namespace llvm { -template <> struct GraphTraits<RematGraph *> { +template <> struct llvm::GraphTraits<RematGraph *> { using NodeRef = RematGraph::RematNode *; using ChildIteratorType = RematGraph::RematNode **; @@ -149,8 +148,6 @@ template <> struct GraphTraits<RematGraph *> { static ChildIteratorType child_end(NodeRef N) { return N->Operands.end(); } }; -} // end namespace llvm - // For each instruction identified as materializable across the suspend point, // and its associated DAG of other rematerializable instructions, // recreate the DAG of instructions after the suspend point. diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp index e474c07..81fe0c9 100644 --- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp +++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp @@ -16,11 +16,8 @@ #include "llvm/IR/InstIterator.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" -namespace llvm { - -namespace coro { - -namespace { +using namespace llvm; +using namespace llvm::coro; typedef SmallPtrSet<BasicBlock *, 8> VisitedBlocksSet; @@ -71,7 +68,7 @@ static bool isLocalAlloca(CoroAllocaAllocInst *AI) { /// This happens during the all-instructions iteration, so it must not /// delete the call. static Instruction * -lowerNonLocalAlloca(CoroAllocaAllocInst *AI, const coro::Shape &Shape, +lowerNonLocalAlloca(CoroAllocaAllocInst *AI, const Shape &Shape, SmallVectorImpl<Instruction *> &DeadInsts) { IRBuilder<> Builder(AI); auto Alloc = Shape.emitAlloc(Builder, AI->getSize(), nullptr); @@ -450,10 +447,8 @@ static void collectFrameAlloca(AllocaInst *AI, const coro::Shape &Shape, Visitor.getMayWriteBeforeCoroBegin()); } -} // namespace - -void collectSpillsFromArgs(SpillInfo &Spills, Function &F, - const SuspendCrossingInfo &Checker) { +void coro::collectSpillsFromArgs(SpillInfo &Spills, Function &F, + const SuspendCrossingInfo &Checker) { // Collect the spills for arguments and other not-materializable values. for (Argument &A : F.args()) for (User *U : A.users()) @@ -461,7 +456,7 @@ void collectSpillsFromArgs(SpillInfo &Spills, Function &F, Spills[&A].push_back(cast<Instruction>(U)); } -void collectSpillsAndAllocasFromInsts( +void coro::collectSpillsAndAllocasFromInsts( SpillInfo &Spills, SmallVector<AllocaInfo, 8> &Allocas, SmallVector<Instruction *, 4> &DeadInstructions, SmallVector<CoroAllocaAllocInst *, 4> &LocalAllocas, Function &F, @@ -516,8 +511,8 @@ void collectSpillsAndAllocasFromInsts( } } -void collectSpillsFromDbgInfo(SpillInfo &Spills, Function &F, - const SuspendCrossingInfo &Checker) { +void coro::collectSpillsFromDbgInfo(SpillInfo &Spills, Function &F, + const SuspendCrossingInfo &Checker) { // We don't want the layout of coroutine frame to be affected // by debug information. So we only choose to salvage dbg.values for // whose value is already in the frame. @@ -535,10 +530,9 @@ void collectSpillsFromDbgInfo(SpillInfo &Spills, Function &F, /// Async and Retcon{Once} conventions assume that all spill uses can be sunk /// after the coro.begin intrinsic. -void sinkSpillUsesAfterCoroBegin(const DominatorTree &Dom, - CoroBeginInst *CoroBegin, - coro::SpillInfo &Spills, - SmallVectorImpl<coro::AllocaInfo> &Allocas) { +void coro::sinkSpillUsesAfterCoroBegin( + const DominatorTree &Dom, CoroBeginInst *CoroBegin, coro::SpillInfo &Spills, + SmallVectorImpl<coro::AllocaInfo> &Allocas) { SmallSetVector<Instruction *, 32> ToMove; SmallVector<Instruction *, 32> Worklist; @@ -582,8 +576,9 @@ void sinkSpillUsesAfterCoroBegin(const DominatorTree &Dom, Inst->moveBefore(InsertPt->getIterator()); } -BasicBlock::iterator getSpillInsertionPt(const coro::Shape &Shape, Value *Def, - const DominatorTree &DT) { +BasicBlock::iterator coro::getSpillInsertionPt(const coro::Shape &Shape, + Value *Def, + const DominatorTree &DT) { BasicBlock::iterator InsertPt; if (auto *Arg = dyn_cast<Argument>(Def)) { // For arguments, we will place the store instruction right after @@ -625,7 +620,3 @@ BasicBlock::iterator getSpillInsertionPt(const coro::Shape &Shape, Value *Def, return InsertPt; } - -} // End namespace coro. - -} // End namespace llvm. diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h index 7071876..943c223 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h +++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h @@ -471,7 +471,6 @@ private: Value *simplifyNonNullOperand(Value *V, bool HasDereferenceable, unsigned Depth = 0); -public: /// Create `select C, S1, S2`. Use only when the profile cannot be calculated /// from existing profile metadata: if the Function has profiles, this will /// set the profile of this select to "unknown". @@ -484,6 +483,7 @@ public: return Sel; } +public: /// Create and insert the idiom we use to indicate a block is unreachable /// without having to rewrite the CFG from within InstCombine. void CreateNonTerminatorUnreachable(Instruction *InsertAt) { diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index 6b67b48..09cb225 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -2979,10 +2979,14 @@ Instruction *InstCombinerImpl::foldAndOrOfSelectUsingImpliedCond(Value *Op, "Op must be either i1 or vector of i1."); if (SI.getCondition()->getType() != Op->getType()) return nullptr; - if (Value *V = simplifyNestedSelectsUsingImpliedCond(SI, Op, IsAnd, DL)) - return SelectInst::Create(Op, - IsAnd ? V : ConstantInt::getTrue(Op->getType()), - IsAnd ? ConstantInt::getFalse(Op->getType()) : V); + if (Value *V = simplifyNestedSelectsUsingImpliedCond(SI, Op, IsAnd, DL)) { + Instruction *MDFrom = nullptr; + if (!ProfcheckDisableMetadataFixes) + MDFrom = &SI; + return SelectInst::Create( + Op, IsAnd ? V : ConstantInt::getTrue(Op->getType()), + IsAnd ? ConstantInt::getFalse(Op->getType()) : V, "", nullptr, MDFrom); + } return nullptr; } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index 63e24a0..a330bb7 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -110,8 +110,8 @@ static Value *simplifyShiftSelectingPackedElement(Instruction *I, ShrAmt->getName() + ".z"); // There is no existing !prof metadata we can derive the !prof metadata for // this select. - Value *Select = IC.createSelectInstWithUnknownProfile(ShrAmtZ, Lower, Upper); - IC.Builder.Insert(Select); + Value *Select = IC.Builder.CreateSelectWithUnknownProfile(ShrAmtZ, Lower, + Upper, DEBUG_TYPE); Select->takeName(I); return Select; } diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 82ac903..3f11cae 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1690,6 +1690,11 @@ Instruction *InstCombinerImpl::foldFBinOpOfIntCastsFromSign( // 2) (fp_binop ({s|u}itofp x), FpC) // -> ({s|u}itofp (int_binop x, (fpto{s|u}i FpC))) Instruction *InstCombinerImpl::foldFBinOpOfIntCasts(BinaryOperator &BO) { + // Don't perform the fold on vectors, as the integer operation may be much + // more expensive than the float operation in that case. + if (BO.getType()->isVectorTy()) + return nullptr; + std::array<Value *, 2> IntOps = {nullptr, nullptr}; Constant *Op1FpC = nullptr; // Check for: diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index 860f8f7..511bca4 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -3337,7 +3337,7 @@ PHINode *FunctionStackPoisoner::createPHI(IRBuilder<> &IRB, Value *Cond, Value *ValueIfTrue, Instruction *ThenTerm, Value *ValueIfFalse) { - PHINode *PHI = IRB.CreatePHI(IntptrTy, 2); + PHINode *PHI = IRB.CreatePHI(ValueIfTrue->getType(), 2); BasicBlock *CondBlock = cast<Instruction>(Cond)->getParent(); PHI->addIncoming(ValueIfFalse, CondBlock); BasicBlock *ThenBlock = ThenTerm->getParent(); @@ -3360,7 +3360,7 @@ Value *FunctionStackPoisoner::createAllocaForLayout( assert((ClRealignStack & (ClRealignStack - 1)) == 0); uint64_t FrameAlignment = std::max(L.FrameAlignment, uint64_t(ClRealignStack)); Alloca->setAlignment(Align(FrameAlignment)); - return IRB.CreatePointerCast(Alloca, IntptrTy); + return Alloca; } void FunctionStackPoisoner::createDynamicAllocasInitStorage() { @@ -3572,10 +3572,12 @@ void FunctionStackPoisoner::processStaticAllocas() { DoDynamicAlloca &= !HasInlineAsm && !HasReturnsTwiceCall; DoStackMalloc &= !HasInlineAsm && !HasReturnsTwiceCall; + Type *PtrTy = F.getDataLayout().getAllocaPtrType(F.getContext()); Value *StaticAlloca = DoDynamicAlloca ? nullptr : createAllocaForLayout(IRB, L, false); - Value *FakeStack; + Value *FakeStackPtr; + Value *FakeStackInt; Value *LocalStackBase; Value *LocalStackBaseAlloca; uint8_t DIExprFlags = DIExpression::ApplyOffset; @@ -3603,20 +3605,21 @@ void FunctionStackPoisoner::processStaticAllocas() { RTCI.createRuntimeCall(IRBIf, AsanStackMallocFunc[StackMallocIdx], ConstantInt::get(IntptrTy, LocalStackSize)); IRB.SetInsertPoint(InsBefore); - FakeStack = createPHI(IRB, UseAfterReturnIsEnabled, FakeStackValue, Term, - ConstantInt::get(IntptrTy, 0)); + FakeStackInt = createPHI(IRB, UseAfterReturnIsEnabled, FakeStackValue, + Term, ConstantInt::get(IntptrTy, 0)); } else { // assert(ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode:Always) // void *FakeStack = __asan_stack_malloc_N(LocalStackSize); // void *LocalStackBase = (FakeStack) ? FakeStack : // alloca(LocalStackSize); StackMallocIdx = StackMallocSizeClass(LocalStackSize); - FakeStack = + FakeStackInt = RTCI.createRuntimeCall(IRB, AsanStackMallocFunc[StackMallocIdx], ConstantInt::get(IntptrTy, LocalStackSize)); } + FakeStackPtr = IRB.CreateIntToPtr(FakeStackInt, PtrTy); Value *NoFakeStack = - IRB.CreateICmpEQ(FakeStack, Constant::getNullValue(IntptrTy)); + IRB.CreateICmpEQ(FakeStackInt, Constant::getNullValue(IntptrTy)); Instruction *Term = SplitBlockAndInsertIfThen(NoFakeStack, InsBefore, false); IRBuilder<> IRBIf(Term); @@ -3624,67 +3627,53 @@ void FunctionStackPoisoner::processStaticAllocas() { DoDynamicAlloca ? createAllocaForLayout(IRBIf, L, true) : StaticAlloca; IRB.SetInsertPoint(InsBefore); - LocalStackBase = createPHI(IRB, NoFakeStack, AllocaValue, Term, FakeStack); + LocalStackBase = + createPHI(IRB, NoFakeStack, AllocaValue, Term, FakeStackPtr); IRB.CreateStore(LocalStackBase, LocalStackBaseAlloca); DIExprFlags |= DIExpression::DerefBefore; } else { // void *FakeStack = nullptr; // void *LocalStackBase = alloca(LocalStackSize); - FakeStack = ConstantInt::get(IntptrTy, 0); + FakeStackInt = Constant::getNullValue(IntptrTy); + FakeStackPtr = Constant::getNullValue(PtrTy); LocalStackBase = DoDynamicAlloca ? createAllocaForLayout(IRB, L, true) : StaticAlloca; LocalStackBaseAlloca = LocalStackBase; } - // It shouldn't matter whether we pass an `alloca` or a `ptrtoint` as the - // dbg.declare address opereand, but passing a `ptrtoint` seems to confuse - // later passes and can result in dropped variable coverage in debug info. - Value *LocalStackBaseAllocaPtr = - isa<PtrToIntInst>(LocalStackBaseAlloca) - ? cast<PtrToIntInst>(LocalStackBaseAlloca)->getPointerOperand() - : LocalStackBaseAlloca; - assert(isa<AllocaInst>(LocalStackBaseAllocaPtr) && - "Variable descriptions relative to ASan stack base will be dropped"); - // Replace Alloca instructions with base+offset. SmallVector<Value *> NewAllocaPtrs; for (const auto &Desc : SVD) { AllocaInst *AI = Desc.AI; - replaceDbgDeclare(AI, LocalStackBaseAllocaPtr, DIB, DIExprFlags, - Desc.Offset); - Value *NewAllocaPtr = IRB.CreateIntToPtr( - IRB.CreateAdd(LocalStackBase, ConstantInt::get(IntptrTy, Desc.Offset)), - AI->getType()); + replaceDbgDeclare(AI, LocalStackBaseAlloca, DIB, DIExprFlags, Desc.Offset); + Value *NewAllocaPtr = IRB.CreatePtrAdd( + LocalStackBase, ConstantInt::get(IntptrTy, Desc.Offset)); AI->replaceAllUsesWith(NewAllocaPtr); NewAllocaPtrs.push_back(NewAllocaPtr); } // The left-most redzone has enough space for at least 4 pointers. // Write the Magic value to redzone[0]. - Value *BasePlus0 = IRB.CreateIntToPtr(LocalStackBase, IntptrPtrTy); IRB.CreateStore(ConstantInt::get(IntptrTy, kCurrentStackFrameMagic), - BasePlus0); + LocalStackBase); // Write the frame description constant to redzone[1]. - Value *BasePlus1 = IRB.CreateIntToPtr( - IRB.CreateAdd(LocalStackBase, - ConstantInt::get(IntptrTy, ASan.LongSize / 8)), - IntptrPtrTy); + Value *BasePlus1 = IRB.CreatePtrAdd( + LocalStackBase, ConstantInt::get(IntptrTy, ASan.LongSize / 8)); GlobalVariable *StackDescriptionGlobal = createPrivateGlobalForString(*F.getParent(), DescriptionString, /*AllowMerging*/ true, genName("stack")); Value *Description = IRB.CreatePointerCast(StackDescriptionGlobal, IntptrTy); IRB.CreateStore(Description, BasePlus1); // Write the PC to redzone[2]. - Value *BasePlus2 = IRB.CreateIntToPtr( - IRB.CreateAdd(LocalStackBase, - ConstantInt::get(IntptrTy, 2 * ASan.LongSize / 8)), - IntptrPtrTy); + Value *BasePlus2 = IRB.CreatePtrAdd( + LocalStackBase, ConstantInt::get(IntptrTy, 2 * ASan.LongSize / 8)); IRB.CreateStore(IRB.CreatePointerCast(&F, IntptrTy), BasePlus2); const auto &ShadowAfterScope = GetShadowBytesAfterScope(SVD, L); // Poison the stack red zones at the entry. - Value *ShadowBase = ASan.memToShadow(LocalStackBase, IRB); + Value *ShadowBase = + ASan.memToShadow(IRB.CreatePtrToInt(LocalStackBase, IntptrTy), IRB); // As mask we must use most poisoned case: red zones and after scope. // As bytes we can use either the same or just red zones only. copyToShadow(ShadowAfterScope, ShadowAfterScope, IRB, ShadowBase); @@ -3723,7 +3712,7 @@ void FunctionStackPoisoner::processStaticAllocas() { IRBuilder<> IRBRet(Ret); // Mark the current frame as retired. IRBRet.CreateStore(ConstantInt::get(IntptrTy, kRetiredStackFrameMagic), - BasePlus0); + LocalStackBase); if (DoStackMalloc) { assert(StackMallocIdx >= 0); // if FakeStack != 0 // LocalStackBase == FakeStack @@ -3737,7 +3726,7 @@ void FunctionStackPoisoner::processStaticAllocas() { // else // <This is not a fake stack; unpoison the redzones> Value *Cmp = - IRBRet.CreateICmpNE(FakeStack, Constant::getNullValue(IntptrTy)); + IRBRet.CreateICmpNE(FakeStackInt, Constant::getNullValue(IntptrTy)); Instruction *ThenTerm, *ElseTerm; SplitBlockAndInsertIfThenElse(Cmp, Ret, &ThenTerm, &ElseTerm); @@ -3748,11 +3737,10 @@ void FunctionStackPoisoner::processStaticAllocas() { kAsanStackUseAfterReturnMagic); copyToShadow(ShadowAfterReturn, ShadowAfterReturn, IRBPoison, ShadowBase); - Value *SavedFlagPtrPtr = IRBPoison.CreateAdd( - FakeStack, + Value *SavedFlagPtrPtr = IRBPoison.CreatePtrAdd( + FakeStackPtr, ConstantInt::get(IntptrTy, ClassSize - ASan.LongSize / 8)); - Value *SavedFlagPtr = IRBPoison.CreateLoad( - IntptrTy, IRBPoison.CreateIntToPtr(SavedFlagPtrPtr, IntptrPtrTy)); + Value *SavedFlagPtr = IRBPoison.CreateLoad(IntptrTy, SavedFlagPtrPtr); IRBPoison.CreateStore( Constant::getNullValue(IRBPoison.getInt8Ty()), IRBPoison.CreateIntToPtr(SavedFlagPtr, IRBPoison.getPtrTy())); @@ -3760,7 +3748,7 @@ void FunctionStackPoisoner::processStaticAllocas() { // For larger frames call __asan_stack_free_*. RTCI.createRuntimeCall( IRBPoison, AsanStackFreeFunc[StackMallocIdx], - {FakeStack, ConstantInt::get(IntptrTy, LocalStackSize)}); + {FakeStackInt, ConstantInt::get(IntptrTy, LocalStackSize)}); } IRBuilder<> IRBElse(ElseTerm); diff --git a/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp b/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp index c86092b..a6ec6c1 100644 --- a/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp @@ -17,6 +17,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/MemoryProfileInfo.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" +#include "llvm/Analysis/StaticDataProfileInfo.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Function.h" @@ -194,6 +195,30 @@ static bool isAllocationWithHotColdVariant(const Function *Callee, } } +static void HandleUnsupportedAnnotationKinds(GlobalVariable &GVar, + AnnotationKind Kind) { + assert(Kind != llvm::memprof::AnnotationKind::AnnotationOK && + "Should not handle AnnotationOK here"); + SmallString<32> Reason; + switch (Kind) { + case llvm::memprof::AnnotationKind::ExplicitSection: + ++NumOfMemProfExplicitSectionGlobalVars; + Reason.append("explicit section name"); + break; + case llvm::memprof::AnnotationKind::DeclForLinker: + Reason.append("linker declaration"); + break; + case llvm::memprof::AnnotationKind::ReservedName: + Reason.append("name starts with `llvm.`"); + break; + default: + llvm_unreachable("Unexpected annotation kind"); + } + LLVM_DEBUG(dbgs() << "Skip annotation for " << GVar.getName() << " due to " + << Reason << ".\n"); + return; +} + struct AllocMatchInfo { uint64_t TotalSize = 0; AllocationType AllocType = AllocationType::None; @@ -775,29 +800,13 @@ PreservedAnalyses MemProfUsePass::run(Module &M, ModuleAnalysisManager &AM) { return PreservedAnalyses::none(); } -// Returns true iff the global variable has custom section either by -// __attribute__((section("name"))) -// (https://clang.llvm.org/docs/AttributeReference.html#section-declspec-allocate) -// or #pragma clang section directives -// (https://clang.llvm.org/docs/LanguageExtensions.html#specifying-section-names-for-global-objects-pragma-clang-section). -static bool hasExplicitSectionName(const GlobalVariable &GVar) { - if (GVar.hasSection()) - return true; - - auto Attrs = GVar.getAttributes(); - if (Attrs.hasAttribute("bss-section") || Attrs.hasAttribute("data-section") || - Attrs.hasAttribute("relro-section") || - Attrs.hasAttribute("rodata-section")) - return true; - return false; -} - bool MemProfUsePass::annotateGlobalVariables( Module &M, const memprof::DataAccessProfData *DataAccessProf) { if (!AnnotateStaticDataSectionPrefix || M.globals().empty()) return false; if (!DataAccessProf) { + M.addModuleFlag(Module::Warning, "EnableDataAccessProf", 0U); M.getContext().diagnose(DiagnosticInfoPGOProfile( MemoryProfileFileName.data(), StringRef("Data access profiles not found in memprof. Ignore " @@ -805,6 +814,7 @@ bool MemProfUsePass::annotateGlobalVariables( DS_Warning)); return false; } + M.addModuleFlag(Module::Warning, "EnableDataAccessProf", 1U); bool Changed = false; // Iterate all global variables in the module and annotate them based on @@ -815,13 +825,9 @@ bool MemProfUsePass::annotateGlobalVariables( for (GlobalVariable &GVar : M.globals()) { assert(!GVar.getSectionPrefix().has_value() && "GVar shouldn't have section prefix yet"); - if (GVar.isDeclarationForLinker()) - continue; - - if (hasExplicitSectionName(GVar)) { - ++NumOfMemProfExplicitSectionGlobalVars; - LLVM_DEBUG(dbgs() << "Global variable " << GVar.getName() - << " has explicit section name. Skip annotating.\n"); + auto Kind = llvm::memprof::getAnnotationKind(GVar); + if (Kind != llvm::memprof::AnnotationKind::AnnotationOK) { + HandleUnsupportedAnnotationKinds(GVar, Kind); continue; } @@ -831,7 +837,6 @@ bool MemProfUsePass::annotateGlobalVariables( // TODO: Track string content hash in the profiles and compute it inside the // compiler to categeorize the hotness string literals. if (Name.starts_with(".str")) { - LLVM_DEBUG(dbgs() << "Skip annotating string literal " << Name << "\n"); continue; } diff --git a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp index ff5f390..66e45ec 100644 --- a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp +++ b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp @@ -266,8 +266,7 @@ void DFAJumpThreading::unfold(DomTreeUpdater *DTU, LoopInfo *LI, if (!ProfcheckDisableMetadataFixes) BI->setMetadata(LLVMContext::MD_prof, SI->getMetadata(LLVMContext::MD_prof)); - DTU->applyUpdates({{DominatorTree::Insert, StartBlock, EndBlock}, - {DominatorTree::Insert, StartBlock, NewBlock}}); + DTU->applyUpdates({{DominatorTree::Insert, StartBlock, NewBlock}}); } else { BasicBlock *EndBlock = SIUse->getParent(); BasicBlock *NewBlockT = BasicBlock::Create( @@ -1479,10 +1478,13 @@ bool DFAJumpThreading::run(Function &F) { DTU->flush(); #ifdef EXPENSIVE_CHECKS - assert(DTU->getDomTree().verify(DominatorTree::VerificationLevel::Full)); verifyFunction(F, &dbgs()); #endif + if (MadeChanges && VerifyDomInfo) + assert(DTU->getDomTree().verify(DominatorTree::VerificationLevel::Full) && + "Failed to maintain validity of domtree!"); + return MadeChanges; } diff --git a/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp b/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp index 213d0f3..1335665 100644 --- a/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp +++ b/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp @@ -39,10 +39,11 @@ public: private: AliasAnalysis *AA; }; +} // namespace /// iterativelyFlattenCFG - Call FlattenCFG on all the blocks in the function, /// iterating until no more changes are made. -bool iterativelyFlattenCFG(Function &F, AliasAnalysis *AA) { +static bool iterativelyFlattenCFG(Function &F, AliasAnalysis *AA) { bool Changed = false; bool LocalChange = true; @@ -67,7 +68,6 @@ bool iterativelyFlattenCFG(Function &F, AliasAnalysis *AA) { } return Changed; } -} // namespace char FlattenCFGLegacyPass::ID = 0; diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp index 56e0569..7cae94eb 100644 --- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp +++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp @@ -1295,6 +1295,24 @@ public: return commonAlignment(InitialAlign, ElementSizeInBits / 8); } + IntegerType *getIndexType(Value *Ptr) const { + return cast<IntegerType>(DL.getIndexType(Ptr->getType())); + } + + Value *getIndex(Value *Ptr, uint64_t V) const { + return ConstantInt::get(getIndexType(Ptr), V); + } + + Value *castToIndexType(Value *Ptr, Value *V, IRBuilder<> &Builder) const { + assert(isa<IntegerType>(V->getType()) && + "Attempted to cast non-integral type to integer index"); + // In case the data layout's index type differs in width from the type of + // the value we're given, truncate or zero extend to the appropriate width. + // We zero extend here as indices are unsigned. + return Builder.CreateZExtOrTrunc(V, getIndexType(Ptr), + V->getName() + ".cast"); + } + /// Load a matrix with \p Shape starting at \p Ptr and using \p Stride between /// vectors. MatrixTy loadMatrix(Type *Ty, Value *Ptr, MaybeAlign MAlign, Value *Stride, @@ -1304,6 +1322,7 @@ public: Type *VecTy = FixedVectorType::get(EltTy, Shape.getStride()); Value *EltPtr = Ptr; MatrixTy Result; + Stride = castToIndexType(Ptr, Stride, Builder); for (unsigned I = 0, E = Shape.getNumVectors(); I < E; ++I) { Value *GEP = computeVectorAddr( EltPtr, Builder.getIntN(Stride->getType()->getScalarSizeInBits(), I), @@ -1325,14 +1344,14 @@ public: ShapeInfo ResultShape, Type *EltTy, IRBuilder<> &Builder) { Value *Offset = Builder.CreateAdd( - Builder.CreateMul(J, Builder.getInt64(MatrixShape.getStride())), I); + Builder.CreateMul(J, getIndex(MatrixPtr, MatrixShape.getStride())), I); Value *TileStart = Builder.CreateGEP(EltTy, MatrixPtr, Offset); auto *TileTy = FixedVectorType::get(EltTy, ResultShape.NumRows * ResultShape.NumColumns); return loadMatrix(TileTy, TileStart, Align, - Builder.getInt64(MatrixShape.getStride()), IsVolatile, + getIndex(MatrixPtr, MatrixShape.getStride()), IsVolatile, ResultShape, Builder); } @@ -1363,14 +1382,15 @@ public: MaybeAlign MAlign, bool IsVolatile, ShapeInfo MatrixShape, Value *I, Value *J, Type *EltTy, IRBuilder<> &Builder) { Value *Offset = Builder.CreateAdd( - Builder.CreateMul(J, Builder.getInt64(MatrixShape.getStride())), I); + Builder.CreateMul(J, getIndex(MatrixPtr, MatrixShape.getStride())), I); Value *TileStart = Builder.CreateGEP(EltTy, MatrixPtr, Offset); auto *TileTy = FixedVectorType::get(EltTy, StoreVal.getNumRows() * StoreVal.getNumColumns()); storeMatrix(TileTy, StoreVal, TileStart, MAlign, - Builder.getInt64(MatrixShape.getStride()), IsVolatile, Builder); + getIndex(MatrixPtr, MatrixShape.getStride()), IsVolatile, + Builder); } /// Store matrix \p StoreVal starting at \p Ptr and using \p Stride between @@ -1380,6 +1400,7 @@ public: IRBuilder<> &Builder) { auto *VType = cast<FixedVectorType>(Ty); Value *EltPtr = Ptr; + Stride = castToIndexType(Ptr, Stride, Builder); for (auto Vec : enumerate(StoreVal.vectors())) { Value *GEP = computeVectorAddr( EltPtr, @@ -2011,18 +2032,17 @@ public: const unsigned TileM = std::min(M - K, unsigned(TileSize)); MatrixTy A = loadMatrix(APtr, LoadOp0->getAlign(), LoadOp0->isVolatile(), - LShape, Builder.getInt64(I), Builder.getInt64(K), + LShape, getIndex(APtr, I), getIndex(APtr, K), {TileR, TileM}, EltType, Builder); MatrixTy B = loadMatrix(BPtr, LoadOp1->getAlign(), LoadOp1->isVolatile(), - RShape, Builder.getInt64(K), Builder.getInt64(J), + RShape, getIndex(BPtr, K), getIndex(BPtr, J), {TileM, TileC}, EltType, Builder); emitMatrixMultiply(Res, A, B, Builder, true, false, getFastMathFlags(MatMul)); } storeMatrix(Res, CPtr, Store->getAlign(), Store->isVolatile(), {R, M}, - Builder.getInt64(I), Builder.getInt64(J), EltType, - Builder); + getIndex(CPtr, I), getIndex(CPtr, J), EltType, Builder); } } @@ -2254,15 +2274,14 @@ public: /// Lower load instructions. MatrixTy VisitLoad(LoadInst *Inst, const ShapeInfo &SI, Value *Ptr, IRBuilder<> &Builder) { - return LowerLoad(Inst, Ptr, Inst->getAlign(), - Builder.getInt64(SI.getStride()), Inst->isVolatile(), SI, - Builder); + return LowerLoad(Inst, Ptr, Inst->getAlign(), getIndex(Ptr, SI.getStride()), + Inst->isVolatile(), SI, Builder); } MatrixTy VisitStore(StoreInst *Inst, const ShapeInfo &SI, Value *StoredVal, Value *Ptr, IRBuilder<> &Builder) { return LowerStore(Inst, StoredVal, Ptr, Inst->getAlign(), - Builder.getInt64(SI.getStride()), Inst->isVolatile(), SI, + getIndex(Ptr, SI.getStride()), Inst->isVolatile(), SI, Builder); } diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index b9d332b..d2f09e9 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -118,9 +118,13 @@ STATISTIC( STATISTIC(NumDeleted, "Number of instructions deleted"); STATISTIC(NumVectorized, "Number of vectorized aggregates"); +namespace llvm { /// Disable running mem2reg during SROA in order to test or debug SROA. static cl::opt<bool> SROASkipMem2Reg("sroa-skip-mem2reg", cl::init(false), cl::Hidden); +extern cl::opt<bool> ProfcheckDisableMetadataFixes; +} // namespace llvm + namespace { class AllocaSliceRewriter; @@ -1777,7 +1781,8 @@ static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI, } Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL, - LI.getName() + ".sroa.speculated"); + LI.getName() + ".sroa.speculated", + ProfcheckDisableMetadataFixes ? nullptr : &SI); LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n"); LI.replaceAllUsesWith(V); @@ -4360,10 +4365,13 @@ private: }; Value *Cond, *True, *False; + Instruction *MDFrom = nullptr; if (auto *SI = dyn_cast<SelectInst>(Sel)) { Cond = SI->getCondition(); True = SI->getTrueValue(); False = SI->getFalseValue(); + if (!ProfcheckDisableMetadataFixes) + MDFrom = SI; } else { Cond = Sel->getOperand(0); True = ConstantInt::get(Sel->getType(), 1); @@ -4383,8 +4391,12 @@ private: IRB.CreateGEP(Ty, FalseOps[0], ArrayRef(FalseOps).drop_front(), False->getName() + ".sroa.gep", NW); - Value *NSel = - IRB.CreateSelect(Cond, NTrue, NFalse, Sel->getName() + ".sroa.sel"); + Value *NSel = MDFrom + ? IRB.CreateSelect(Cond, NTrue, NFalse, + Sel->getName() + ".sroa.sel", MDFrom) + : IRB.CreateSelectWithUnknownProfile( + Cond, NTrue, NFalse, DEBUG_TYPE, + Sel->getName() + ".sroa.sel"); Visited.erase(&GEPI); GEPI.replaceAllUsesWith(NSel); GEPI.eraseFromParent(); diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp index 2ee91a9..0f3978f 100644 --- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp +++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp @@ -47,6 +47,7 @@ #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/SSAUpdater.h" +#include "llvm/Transforms/Utils/SSAUpdaterBulk.h" #include <cassert> #include <utility> @@ -321,7 +322,7 @@ class StructurizeCFG { void collectInfos(); - void insertConditions(bool Loops); + void insertConditions(bool Loops, SSAUpdaterBulk &PhiInserter); void simplifyConditions(); @@ -671,10 +672,9 @@ void StructurizeCFG::collectInfos() { } /// Insert the missing branch conditions -void StructurizeCFG::insertConditions(bool Loops) { +void StructurizeCFG::insertConditions(bool Loops, SSAUpdaterBulk &PhiInserter) { BranchVector &Conds = Loops ? LoopConds : Conditions; Value *Default = Loops ? BoolTrue : BoolFalse; - SSAUpdater PhiInserter; for (BranchInst *Term : Conds) { assert(Term->isConditional()); @@ -683,8 +683,9 @@ void StructurizeCFG::insertConditions(bool Loops) { BasicBlock *SuccTrue = Term->getSuccessor(0); BasicBlock *SuccFalse = Term->getSuccessor(1); - PhiInserter.Initialize(Boolean, ""); - PhiInserter.AddAvailableValue(Loops ? SuccFalse : Parent, Default); + unsigned Variable = PhiInserter.AddVariable("", Boolean); + PhiInserter.AddAvailableValue(Variable, Loops ? SuccFalse : Parent, + Default); BBPredicates &Preds = Loops ? LoopPreds[SuccFalse] : Predicates[SuccTrue]; @@ -697,7 +698,7 @@ void StructurizeCFG::insertConditions(bool Loops) { ParentInfo = PI; break; } - PhiInserter.AddAvailableValue(BB, PI.Pred); + PhiInserter.AddAvailableValue(Variable, BB, PI.Pred); Dominator.addAndRememberBlock(BB); } @@ -706,9 +707,9 @@ void StructurizeCFG::insertConditions(bool Loops) { CondBranchWeights::setMetadata(*Term, ParentInfo.Weights); } else { if (!Dominator.resultIsRememberedBlock()) - PhiInserter.AddAvailableValue(Dominator.result(), Default); + PhiInserter.AddAvailableValue(Variable, Dominator.result(), Default); - Term->setCondition(PhiInserter.GetValueInMiddleOfBlock(Parent)); + PhiInserter.AddUse(Variable, &Term->getOperandUse(0)); } } } @@ -1414,8 +1415,12 @@ bool StructurizeCFG::run(Region *R, DominatorTree *DT, orderNodes(); collectInfos(); createFlow(); - insertConditions(false); - insertConditions(true); + + SSAUpdaterBulk PhiInserter; + insertConditions(false, PhiInserter); + insertConditions(true, PhiInserter); + PhiInserter.RewriteAndOptimizeAllUses(*DT); + setPhiValues(); simplifyHoistedPhis(); simplifyConditions(); diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp index b187208..32924e7 100644 --- a/llvm/lib/Transforms/Utils/CloneFunction.cpp +++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp @@ -44,7 +44,7 @@ using namespace llvm; STATISTIC(RemappedAtomMax, "Highest global NextAtomGroup (after mapping)"); void llvm::mapAtomInstance(const DebugLoc &DL, ValueToValueMapTy &VMap) { - auto CurGroup = DL->getAtomGroup(); + uint64_t CurGroup = DL->getAtomGroup(); if (!CurGroup) return; @@ -62,21 +62,20 @@ void llvm::mapAtomInstance(const DebugLoc &DL, ValueToValueMapTy &VMap) { RemappedAtomMax = std::max<uint64_t>(NewGroup, RemappedAtomMax); } -namespace { -void collectDebugInfoFromInstructions(const Function &F, - DebugInfoFinder &DIFinder) { +static void collectDebugInfoFromInstructions(const Function &F, + DebugInfoFinder &DIFinder) { const Module *M = F.getParent(); - if (M) { - // Inspect instructions to process e.g. DILexicalBlocks of inlined functions - for (const auto &I : instructions(F)) - DIFinder.processInstruction(*M, I); - } + if (!M) + return; + // Inspect instructions to process e.g. DILexicalBlocks of inlined functions + for (const Instruction &I : instructions(F)) + DIFinder.processInstruction(*M, I); } // Create a predicate that matches the metadata that should be identity mapped // during function cloning. -MetadataPredicate createIdentityMDPredicate(const Function &F, - CloneFunctionChangeType Changes) { +static MetadataPredicate +createIdentityMDPredicate(const Function &F, CloneFunctionChangeType Changes) { if (Changes >= CloneFunctionChangeType::DifferentModule) return [](const Metadata *MD) { return false; }; @@ -107,7 +106,6 @@ MetadataPredicate createIdentityMDPredicate(const Function &F, return false; }; } -} // namespace /// See comments in Cloning.h. BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap, @@ -213,10 +211,9 @@ void llvm::CloneFunctionMetadataInto(Function &NewFunc, const Function &OldFunc, const MetadataPredicate *IdentityMD) { SmallVector<std::pair<unsigned, MDNode *>, 1> MDs; OldFunc.getAllMetadata(MDs); - for (auto MD : MDs) { - NewFunc.addMetadata(MD.first, - *MapMetadata(MD.second, VMap, RemapFlag, TypeMapper, - Materializer, IdentityMD)); + for (const auto &[Kind, MD] : MDs) { + NewFunc.addMetadata(Kind, *MapMetadata(MD, VMap, RemapFlag, TypeMapper, + Materializer, IdentityMD)); } } @@ -235,7 +232,6 @@ void llvm::CloneFunctionBodyInto(Function &NewFunc, const Function &OldFunc, // appropriate. Note that we save BE this way in order to handle cloning of // recursive functions into themselves. for (const BasicBlock &BB : OldFunc) { - // Create a new basic block and copy instructions into it! BasicBlock *CBB = CloneBasicBlock(&BB, VMap, NameSuffix, &NewFunc, CodeInfo); @@ -321,7 +317,7 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, // Cloning is always a Module level operation, since Metadata needs to be // cloned. - const auto RemapFlag = RF_None; + const RemapFlags RemapFlag = RF_None; CloneFunctionMetadataInto(*NewFunc, *OldFunc, VMap, RemapFlag, TypeMapper, Materializer, &IdentityMD); @@ -346,8 +342,8 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, // visiting the metadata attached to global values, which would allow this // code to be deleted. Alternatively, perhaps give responsibility for this // update to CloneFunctionInto's callers. - auto *NewModule = NewFunc->getParent(); - auto *NMD = NewModule->getOrInsertNamedMetadata("llvm.dbg.cu"); + Module *NewModule = NewFunc->getParent(); + NamedMDNode *NMD = NewModule->getOrInsertNamedMetadata("llvm.dbg.cu"); // Avoid multiple insertions of the same DICompileUnit to NMD. SmallPtrSet<const void *, 8> Visited(llvm::from_range, NMD->operands()); @@ -355,7 +351,7 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, // the function (e.g. as instructions' scope). DebugInfoFinder DIFinder; collectDebugInfoFromInstructions(*OldFunc, DIFinder); - for (auto *Unit : DIFinder.compile_units()) { + for (DICompileUnit *Unit : DIFinder.compile_units()) { MDNode *MappedUnit = MapMetadata(Unit, VMap, RF_None, TypeMapper, Materializer); if (Visited.insert(MappedUnit).second) @@ -821,17 +817,16 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc, --PredCount[Pred]; // Figure out how many entries to remove from each PHI. - for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) - ++PredCount[PN->getIncomingBlock(i)]; + for (BasicBlock *Pred : PN->blocks()) + ++PredCount[Pred]; // At this point, the excess predecessor entries are positive in the // map. Loop over all of the PHIs and remove excess predecessor // entries. BasicBlock::iterator I = NewBB->begin(); for (; (PN = dyn_cast<PHINode>(I)); ++I) { - for (const auto &PCI : PredCount) { - BasicBlock *Pred = PCI.first; - for (unsigned NumToRemove = PCI.second; NumToRemove; --NumToRemove) + for (const auto &[Pred, Count] : PredCount) { + for ([[maybe_unused]] unsigned _ : llvm::seq<unsigned>(Count)) PN->removeIncomingValue(Pred, false); } } @@ -866,8 +861,8 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc, // As phi-nodes have been now remapped, allow incremental simplification of // newly-cloned instructions. const DataLayout &DL = NewFunc->getDataLayout(); - for (const auto &BB : *OldFunc) { - for (const auto &I : BB) { + for (const BasicBlock &BB : *OldFunc) { + for (const Instruction &I : BB) { auto *NewI = dyn_cast_or_null<Instruction>(VMap.lookup(&I)); if (!NewI) continue; @@ -997,8 +992,8 @@ void llvm::CloneAndPruneFunctionInto( void llvm::remapInstructionsInBlocks(ArrayRef<BasicBlock *> Blocks, ValueToValueMapTy &VMap) { // Rewrite the code to refer to itself. - for (auto *BB : Blocks) { - for (auto &Inst : *BB) { + for (BasicBlock *BB : Blocks) { + for (Instruction &Inst : *BB) { RemapDbgRecordRange(Inst.getModule(), Inst.getDbgRecordRange(), VMap, RF_NoModuleLevelChanges | RF_IgnoreMissingLocals); RemapInstruction(&Inst, VMap, @@ -1151,9 +1146,9 @@ void llvm::cloneNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes, StringRef Ext, LLVMContext &Context) { MDBuilder MDB(Context); - for (auto *ScopeList : NoAliasDeclScopes) { - for (const auto &MDOperand : ScopeList->operands()) { - if (MDNode *MD = dyn_cast<MDNode>(MDOperand)) { + for (MDNode *ScopeList : NoAliasDeclScopes) { + for (const MDOperand &MDOp : ScopeList->operands()) { + if (MDNode *MD = dyn_cast<MDNode>(MDOp)) { AliasScopeNode SNANode(MD); std::string Name; @@ -1177,7 +1172,7 @@ void llvm::adaptNoAliasScopes(Instruction *I, auto CloneScopeList = [&](const MDNode *ScopeList) -> MDNode * { bool NeedsReplacement = false; SmallVector<Metadata *, 8> NewScopeList; - for (const auto &MDOp : ScopeList->operands()) { + for (const MDOperand &MDOp : ScopeList->operands()) { if (MDNode *MD = dyn_cast<MDNode>(MDOp)) { if (auto *NewMD = ClonedScopes.lookup(MD)) { NewScopeList.push_back(NewMD); @@ -1193,12 +1188,12 @@ void llvm::adaptNoAliasScopes(Instruction *I, }; if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(I)) - if (auto *NewScopeList = CloneScopeList(Decl->getScopeList())) + if (MDNode *NewScopeList = CloneScopeList(Decl->getScopeList())) Decl->setScopeList(NewScopeList); auto replaceWhenNeeded = [&](unsigned MD_ID) { if (const MDNode *CSNoAlias = I->getMetadata(MD_ID)) - if (auto *NewScopeList = CloneScopeList(CSNoAlias)) + if (MDNode *NewScopeList = CloneScopeList(CSNoAlias)) I->setMetadata(MD_ID, NewScopeList); }; replaceWhenNeeded(LLVMContext::MD_noalias); diff --git a/llvm/lib/Transforms/Utils/SSAUpdaterBulk.cpp b/llvm/lib/Transforms/Utils/SSAUpdaterBulk.cpp index d7bf791..fb39fdd 100644 --- a/llvm/lib/Transforms/Utils/SSAUpdaterBulk.cpp +++ b/llvm/lib/Transforms/Utils/SSAUpdaterBulk.cpp @@ -11,11 +11,11 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/SSAUpdaterBulk.h" +#include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/IteratedDominanceFrontier.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" -#include "llvm/IR/Instructions.h" #include "llvm/IR/Use.h" #include "llvm/IR/Value.h" @@ -112,7 +112,7 @@ struct BBValueInfo { void SSAUpdaterBulk::RewriteAllUses(DominatorTree *DT, SmallVectorImpl<PHINode *> *InsertedPHIs) { DenseMap<BasicBlock *, BBValueInfo> BBInfos; - for (auto &R : Rewrites) { + for (RewriteInfo &R : Rewrites) { BBInfos.clear(); // Compute locations for new phi-nodes. @@ -145,7 +145,7 @@ void SSAUpdaterBulk::RewriteAllUses(DominatorTree *DT, BBInfos[BB].LiveOutValue = V; // We've computed IDF, now insert new phi-nodes there. - for (auto *FrontierBB : IDFBlocks) { + for (BasicBlock *FrontierBB : IDFBlocks) { IRBuilder<> B(FrontierBB, FrontierBB->begin()); PHINode *PN = B.CreatePHI(R.Ty, 0, R.Name); BBInfos[FrontierBB].LiveInValue = PN; @@ -156,7 +156,7 @@ void SSAUpdaterBulk::RewriteAllUses(DominatorTree *DT, // IsLiveOut indicates whether we are computing live-out values (true) or // live-in values (false). auto ComputeValue = [&](BasicBlock *BB, bool IsLiveOut) -> Value * { - auto *BBInfo = &BBInfos[BB]; + BBValueInfo *BBInfo = &BBInfos[BB]; if (IsLiveOut && BBInfo->LiveOutValue) return BBInfo->LiveOutValue; @@ -187,7 +187,7 @@ void SSAUpdaterBulk::RewriteAllUses(DominatorTree *DT, if (!V) V = UndefValue::get(R.Ty); - for (auto *BBInfo : Stack) + for (BBValueInfo *BBInfo : Stack) // Loop above can insert new entries into the BBInfos map: assume the // map shouldn't grow due to [1] and BBInfo references are valid. BBInfo->LiveInValue = V; @@ -196,7 +196,7 @@ void SSAUpdaterBulk::RewriteAllUses(DominatorTree *DT, }; // Fill in arguments of the inserted PHIs. - for (auto *BB : IDFBlocks) { + for (BasicBlock *BB : IDFBlocks) { auto *PHI = cast<PHINode>(&BB->front()); for (BasicBlock *Pred : PredCache.get(BB)) PHI->addIncoming(ComputeValue(Pred, /*IsLiveOut=*/true), Pred); @@ -222,3 +222,96 @@ void SSAUpdaterBulk::RewriteAllUses(DominatorTree *DT, } } } + +// Perform a single pass of simplification over the worklist of PHIs. +// This should be called after RewriteAllUses() because simplifying PHIs +// immediately after creation would require updating all references to those +// PHIs in the BBValueInfo structures, which would necessitate additional +// reference tracking overhead. +static void simplifyPass(MutableArrayRef<PHINode *> Worklist, + const DataLayout &DL) { + for (PHINode *&PHI : Worklist) { + if (Value *Simplified = simplifyInstruction(PHI, DL)) { + PHI->replaceAllUsesWith(Simplified); + PHI->eraseFromParent(); + PHI = nullptr; // Mark as removed. + } + } +} + +#ifndef NDEBUG // Should this be under EXPENSIVE_CHECKS? +// New PHI nodes should not reference one another but they may reference +// themselves or existing PHI nodes, and existing PHI nodes may reference new +// PHI nodes. +static bool +PHIAreRefEachOther(const iterator_range<BasicBlock::phi_iterator> NewPHIs) { + SmallPtrSet<PHINode *, 8> NewPHISet; + for (PHINode &PN : NewPHIs) + NewPHISet.insert(&PN); + for (PHINode &PHI : NewPHIs) { + for (Value *V : PHI.incoming_values()) { + PHINode *IncPHI = dyn_cast<PHINode>(V); + if (IncPHI && IncPHI != &PHI && NewPHISet.contains(IncPHI)) + return true; + } + } + return false; +} +#endif + +static bool replaceIfIdentical(PHINode &PHI, PHINode &ReplPHI) { + if (!PHI.isIdenticalToWhenDefined(&ReplPHI)) + return false; + PHI.replaceAllUsesWith(&ReplPHI); + PHI.eraseFromParent(); + return true; +} + +bool EliminateNewDuplicatePHINodes(BasicBlock *BB, + BasicBlock::phi_iterator FirstExistingPN) { + assert(!PHIAreRefEachOther(make_range(BB->phis().begin(), FirstExistingPN))); + + // Deduplicate new PHIs first to reduce the number of comparisons on the + // following new -> existing pass. + bool Changed = false; + for (auto I = BB->phis().begin(); I != FirstExistingPN; ++I) { + for (auto J = std::next(I); J != FirstExistingPN;) { + Changed |= replaceIfIdentical(*J++, *I); + } + } + + // Iterate over existing PHIs and replace identical new PHIs. + for (PHINode &ExistingPHI : make_range(FirstExistingPN, BB->phis().end())) { + auto I = BB->phis().begin(); + assert(I != FirstExistingPN); // Should be at least one new PHI. + do { + Changed |= replaceIfIdentical(*I++, ExistingPHI); + } while (I != FirstExistingPN); + if (BB->phis().begin() == FirstExistingPN) + return Changed; + } + return Changed; +} + +static void deduplicatePass(ArrayRef<PHINode *> Worklist) { + SmallDenseMap<BasicBlock *, unsigned> BBs; + for (PHINode *PHI : Worklist) { + if (PHI) + ++BBs[PHI->getParent()]; + } + + for (auto [BB, NumNewPHIs] : BBs) { + auto FirstExistingPN = std::next(BB->phis().begin(), NumNewPHIs); + EliminateNewDuplicatePHINodes(BB, FirstExistingPN); + } +} + +void SSAUpdaterBulk::RewriteAndOptimizeAllUses(DominatorTree &DT) { + SmallVector<PHINode *, 4> PHIs; + RewriteAllUses(&DT, &PHIs); + if (PHIs.empty()) + return; + + simplifyPass(PHIs, PHIs.front()->getParent()->getDataLayout()); + deduplicatePass(PHIs); +} diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index 45cee1e..9035e58 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -526,7 +526,7 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) { // Recognize the canonical representation of an unsimplifed urem. const SCEV *URemLHS = nullptr; const SCEV *URemRHS = nullptr; - if (SE.matchURem(S, URemLHS, URemRHS)) { + if (match(S, m_scev_URem(m_SCEV(URemLHS), m_SCEV(URemRHS), SE))) { Value *LHS = expand(URemLHS); Value *RHS = expand(URemRHS); return InsertBinop(Instruction::URem, LHS, RHS, SCEV::FlagAnyWrap, diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index a6f4bec..88af2cf 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -10659,7 +10659,8 @@ class InstructionsCompatibilityAnalysis { static bool isSupportedOpcode(const unsigned Opcode) { return Opcode == Instruction::Add || Opcode == Instruction::LShr || Opcode == Instruction::Shl || Opcode == Instruction::SDiv || - Opcode == Instruction::UDiv; + Opcode == Instruction::UDiv || Opcode == Instruction::And || + Opcode == Instruction::Or || Opcode == Instruction::Xor; } /// Identifies the best candidate value, which represents main opcode @@ -10984,6 +10985,9 @@ public: case Instruction::Shl: case Instruction::SDiv: case Instruction::UDiv: + case Instruction::And: + case Instruction::Or: + case Instruction::Xor: VectorCost = TTI.getArithmeticInstrCost(MainOpcode, VecTy, Kind); break; default: @@ -19456,7 +19460,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) { } assert(getNumElements(Cond->getType()) == TrueNumElements && "Cannot vectorize Instruction::Select"); - Value *V = Builder.CreateSelect(Cond, True, False); + Value *V = + Builder.CreateSelectWithUnknownProfile(Cond, True, False, DEBUG_TYPE); V = FinalShuffle(V, E); E->VectorizedValue = V; @@ -23576,18 +23581,19 @@ class HorizontalReduction { switch (Kind) { case RecurKind::Or: { if (UseSelect && OpTy == CmpInst::makeCmpResultType(OpTy)) - return Builder.CreateSelect( + return Builder.CreateSelectWithUnknownProfile( LHS, ConstantInt::getAllOnesValue(CmpInst::makeCmpResultType(OpTy)), - RHS, Name); + RHS, DEBUG_TYPE, Name); unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind); return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS, Name); } case RecurKind::And: { if (UseSelect && OpTy == CmpInst::makeCmpResultType(OpTy)) - return Builder.CreateSelect( + return Builder.CreateSelectWithUnknownProfile( LHS, RHS, - ConstantInt::getNullValue(CmpInst::makeCmpResultType(OpTy)), Name); + ConstantInt::getNullValue(CmpInst::makeCmpResultType(OpTy)), + DEBUG_TYPE, Name); unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind); return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS, Name); @@ -23608,7 +23614,8 @@ class HorizontalReduction { if (UseSelect) { CmpInst::Predicate Pred = llvm::getMinMaxReductionPredicate(Kind); Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS, Name); - return Builder.CreateSelect(Cmp, LHS, RHS, Name); + return Builder.CreateSelectWithUnknownProfile(Cmp, LHS, RHS, DEBUG_TYPE, + Name); } [[fallthrough]]; case RecurKind::FMax: diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 1fea068..0101942 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -635,9 +635,9 @@ static bool hasConditionalTerminator(const VPBasicBlock *VPBB) { const VPRecipeBase *R = &VPBB->back(); bool IsSwitch = isa<VPInstruction>(R) && cast<VPInstruction>(R)->getOpcode() == Instruction::Switch; - bool IsCondBranch = isa<VPBranchOnMaskRecipe>(R) || - match(R, m_BranchOnCond(m_VPValue())) || - match(R, m_BranchOnCount(m_VPValue(), m_VPValue())); + bool IsCondBranch = + isa<VPBranchOnMaskRecipe>(R) || + match(R, m_CombineOr(m_BranchOnCond(), m_BranchOnCount())); (void)IsCondBranch; (void)IsSwitch; if (VPBB->getNumSuccessors() == 2 || diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index fb696be..23f5623 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -24,12 +24,9 @@ #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H #define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H -#include "VPlanAnalysis.h" #include "VPlanValue.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/ilist.h" @@ -41,10 +38,11 @@ #include "llvm/IR/Operator.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/InstructionCost.h" -#include <algorithm> #include <cassert> #include <cstddef> +#include <functional> #include <string> +#include <utility> namespace llvm { @@ -346,13 +344,6 @@ public: /// Return the cost of the block. virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx) = 0; - /// Return true if it is legal to hoist instructions into this block. - bool isLegalToHoistInto() { - // There are currently no constraints that prevent an instruction to be - // hoisted into a VPBlockBase. - return true; - } - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void printAsOperand(raw_ostream &OS, bool PrintType = false) const { OS << getName(); @@ -1064,6 +1055,7 @@ public: ResumeForEpilogue, /// Returns the value for vscale. VScale, + OpsEnd = VScale, }; /// Returns true if this VPInstruction generates scalar values for all lanes. diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 81deba2..c0147ce 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -433,8 +433,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, VPBasicBlock *HeaderVPBB, // We are about to replace the branch to exit the region. Remove the original // BranchOnCond, if there is any. DebugLoc LatchDL = DL; - if (!LatchVPBB->empty() && - match(&LatchVPBB->back(), m_BranchOnCond(m_VPValue()))) { + if (!LatchVPBB->empty() && match(&LatchVPBB->back(), m_BranchOnCond())) { LatchDL = LatchVPBB->getTerminator()->getDebugLoc(); LatchVPBB->getTerminator()->eraseFromParent(); } @@ -480,8 +479,7 @@ static void createExtractsForLiveOuts(VPlan &Plan, VPBasicBlock *MiddleVPBB) { static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL, PredicatedScalarEvolution &PSE, Loop *TheLoop) { - VPDominatorTree VPDT; - VPDT.recalculate(Plan); + VPDominatorTree VPDT(Plan); auto *HeaderVPBB = cast<VPBasicBlock>(Plan.getEntry()->getSingleSuccessor()); canonicalHeaderAndLatch(HeaderVPBB, VPDT); @@ -623,8 +621,7 @@ void VPlanTransforms::addMiddleCheck(VPlan &Plan, } void VPlanTransforms::createLoopRegions(VPlan &Plan) { - VPDominatorTree VPDT; - VPDT.recalculate(Plan); + VPDominatorTree VPDT(Plan); for (VPBlockBase *HeaderVPB : vp_post_order_shallow(Plan.getEntry())) if (canonicalHeaderAndLatch(HeaderVPB, VPDT)) createLoopRegion(Plan, HeaderVPB); @@ -875,8 +872,7 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) { Plan.getVectorLoopRegion()->getEntryBasicBlock())) { auto *VPBB = cast<VPBasicBlock>(VPB); for (auto &R : *VPBB) { - if (R.mayWriteToMemory() && - !match(&R, m_BranchOnCount(m_VPValue(), m_VPValue()))) + if (R.mayWriteToMemory() && !match(&R, m_BranchOnCount())) return false; } } diff --git a/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h b/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h index 577432f..44506f5a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h +++ b/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h @@ -39,7 +39,6 @@ class VPDominatorTree : public DominatorTreeBase<VPBlockBase, false> { using Base = DominatorTreeBase<VPBlockBase, false>; public: - VPDominatorTree() = default; explicit VPDominatorTree(VPlan &Plan) { recalculate(Plan); } /// Returns true if \p A properly dominates \p B. diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index 555efea..b42b049 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -344,6 +344,10 @@ m_Freeze(const Op0_t &Op0) { return m_VPInstruction<Instruction::Freeze>(Op0); } +inline VPInstruction_match<VPInstruction::BranchOnCond> m_BranchOnCond() { + return m_VPInstruction<VPInstruction::BranchOnCond>(); +} + template <typename Op0_t> inline VPInstruction_match<VPInstruction::BranchOnCond, Op0_t> m_BranchOnCond(const Op0_t &Op0) { @@ -374,6 +378,10 @@ m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) { return m_VPInstruction<VPInstruction::ActiveLaneMask>(Op0, Op1, Op2); } +inline VPInstruction_match<VPInstruction::BranchOnCount> m_BranchOnCount() { + return m_VPInstruction<VPInstruction::BranchOnCount>(); +} + template <typename Op0_t, typename Op1_t> inline VPInstruction_match<VPInstruction::BranchOnCount, Op0_t, Op1_t> m_BranchOnCount(const Op0_t &Op0, const Op1_t &Op1) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 9bb8820..40b7e8d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1658,7 +1658,7 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF, auto *Term = &ExitingVPBB->back(); VPValue *Cond; ScalarEvolution &SE = *PSE.getSE(); - if (match(Term, m_BranchOnCount(m_VPValue(), m_VPValue())) || + if (match(Term, m_BranchOnCount()) || match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask( m_VPValue(), m_VPValue(), m_VPValue()))))) { // Try to simplify the branch condition if TC <= VF * UF when the latch @@ -1909,8 +1909,7 @@ static bool hoistPreviousBeforeFORUsers(VPFirstOrderRecurrencePHIRecipe *FOR, bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &LoopBuilder) { - VPDominatorTree VPDT; - VPDT.recalculate(Plan); + VPDominatorTree VPDT(Plan); SmallVector<VPFirstOrderRecurrencePHIRecipe *> RecurrencePhis; for (VPRecipeBase &R : @@ -1992,6 +1991,13 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> { .Case<VPWidenIntrinsicRecipe>([](auto *I) { return std::make_pair(true, I->getVectorIntrinsicID()); }) + .Case<VPVectorPointerRecipe>([](auto *I) { + // For recipes that do not directly map to LLVM IR instructions, + // assign opcodes after the last VPInstruction opcode (which is also + // after the last IR Instruction opcode), based on the VPDefID. + return std::make_pair(false, + VPInstruction::OpsEnd + 1 + I->getVPDefID()); + }) .Default([](auto *) { return std::nullopt; }); } @@ -2015,11 +2021,8 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> { static bool canHandle(const VPSingleDefRecipe *Def) { // We can extend the list of handled recipes in the future, // provided we account for the data embedded in them while checking for - // equality or hashing. We assign VPVectorEndPointerRecipe the GEP opcode, - // as it is essentially a GEP with different semantics. - auto C = isa<VPVectorPointerRecipe>(Def) - ? std::make_pair(false, Instruction::GetElementPtr) - : getOpcodeOrIntrinsicID(Def); + // equality or hashing. + auto C = getOpcodeOrIntrinsicID(Def); // The issue with (Insert|Extract)Value is that the index of the // insert/extract is not a proper operand in LLVM IR, and hence also not in @@ -2058,6 +2061,8 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> { vputils::isSingleScalar(L) != vputils::isSingleScalar(R) || !equal(L->operands(), R->operands())) return false; + assert(getOpcodeOrIntrinsicID(L) && getOpcodeOrIntrinsicID(R) && + "must have valid opcode info for both recipes"); if (auto *LFlags = dyn_cast<VPRecipeWithIRFlags>(L)) if (LFlags->hasPredicate() && LFlags->getPredicate() != @@ -3021,8 +3026,7 @@ void VPlanTransforms::createInterleaveGroups( // Interleave memory: for each Interleave Group we marked earlier as relevant // for this VPlan, replace the Recipes widening its memory instructions with a // single VPInterleaveRecipe at its insertion point. - VPDominatorTree VPDT; - VPDT.recalculate(Plan); + VPDominatorTree VPDT(Plan); for (const auto *IG : InterleaveGroups) { auto *Start = cast<VPWidenMemoryRecipe>(RecipeBuilder.getRecipe(IG->getMember(0))); @@ -3398,9 +3402,8 @@ void VPlanTransforms::handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB, VPBuilder Builder(LatchVPBB->getTerminator()); VPBlockBase *TrueSucc = EarlyExitingVPBB->getSuccessors()[0]; - assert( - match(EarlyExitingVPBB->getTerminator(), m_BranchOnCond(m_VPValue())) && - "Terminator must be be BranchOnCond"); + assert(match(EarlyExitingVPBB->getTerminator(), m_BranchOnCond()) && + "Terminator must be be BranchOnCond"); VPValue *CondOfEarlyExitingVPBB = EarlyExitingVPBB->getTerminator()->getOperand(0); auto *CondToEarlyExit = TrueSucc == EarlyExitVPBB @@ -3662,8 +3665,7 @@ void VPlanTransforms::materializeBroadcasts(VPlan &Plan) { return; #ifndef NDEBUG - VPDominatorTree VPDT; - VPDT.recalculate(Plan); + VPDominatorTree VPDT(Plan); #endif SmallVector<VPValue *> VPValues; @@ -4009,8 +4011,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF, unsigned VFMinVal = VF.getKnownMinValue(); SmallVector<VPInterleaveRecipe *> StoreGroups; for (auto &R : *VectorLoop->getEntryBasicBlock()) { - if (isa<VPCanonicalIVPHIRecipe>(&R) || - match(&R, m_BranchOnCount(m_VPValue(), m_VPValue()))) + if (isa<VPCanonicalIVPHIRecipe>(&R) || match(&R, m_BranchOnCount())) continue; if (isa<VPDerivedIVRecipe, VPScalarIVStepsRecipe>(&R) && diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index 5e7f19f..1c4adfc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -259,8 +259,7 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R, /// Handle non-header-phi recipes. void UnrollState::unrollRecipeByUF(VPRecipeBase &R) { - if (match(&R, m_BranchOnCond(m_VPValue())) || - match(&R, m_BranchOnCount(m_VPValue(), m_VPValue()))) + if (match(&R, m_CombineOr(m_BranchOnCond(), m_BranchOnCount()))) return; if (auto *VPI = dyn_cast<VPInstruction>(&R)) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 013ea2e..5262af6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -24,6 +24,7 @@ #define DEBUG_TYPE "loop-vectorize" using namespace llvm; +using namespace VPlanPatternMatch; namespace { class VPlanVerifier { @@ -198,7 +199,6 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const { } // EVLIVIncrement is only used by EVLIV & BranchOnCount. // Having more than two users is unexpected. - using namespace llvm::VPlanPatternMatch; if (I->getOpcode() != VPInstruction::Broadcast && I->getNumUsers() != 1 && (I->getNumUsers() != 2 || @@ -479,8 +479,7 @@ bool VPlanVerifier::verify(const VPlan &Plan) { } auto *LastInst = dyn_cast<VPInstruction>(std::prev(Exiting->end())); - if (!LastInst || (LastInst->getOpcode() != VPInstruction::BranchOnCount && - LastInst->getOpcode() != VPInstruction::BranchOnCond)) { + if (!match(LastInst, m_CombineOr(m_BranchOnCond(), m_BranchOnCount()))) { errs() << "VPlan vector loop exit must end with BranchOnCount or " "BranchOnCond VPInstruction\n"; return false; @@ -490,8 +489,7 @@ bool VPlanVerifier::verify(const VPlan &Plan) { } bool llvm::verifyVPlanIsValid(const VPlan &Plan, bool VerifyLate) { - VPDominatorTree VPDT; - VPDT.recalculate(const_cast<VPlan &>(Plan)); + VPDominatorTree VPDT(const_cast<VPlan &>(Plan)); VPTypeAnalysis TypeInfo(Plan); VPlanVerifier Verifier(VPDT, TypeInfo, VerifyLate); return Verifier.verify(Plan); |