diff options
Diffstat (limited to 'llvm/lib/Transforms')
-rw-r--r-- | llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp | 12 | ||||
-rw-r--r-- | llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp | 72 | ||||
-rw-r--r-- | llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp | 8 | ||||
-rw-r--r-- | llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp | 4 | ||||
-rw-r--r-- | llvm/lib/Transforms/Scalar/SROA.cpp | 18 | ||||
-rw-r--r-- | llvm/lib/Transforms/Scalar/StructurizeCFG.cpp | 25 | ||||
-rw-r--r-- | llvm/lib/Transforms/Utils/CloneFunction.cpp | 2 | ||||
-rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlan.h | 13 |
8 files changed, 78 insertions, 76 deletions
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index 6b67b48..09cb225 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -2979,10 +2979,14 @@ Instruction *InstCombinerImpl::foldAndOrOfSelectUsingImpliedCond(Value *Op, "Op must be either i1 or vector of i1."); if (SI.getCondition()->getType() != Op->getType()) return nullptr; - if (Value *V = simplifyNestedSelectsUsingImpliedCond(SI, Op, IsAnd, DL)) - return SelectInst::Create(Op, - IsAnd ? V : ConstantInt::getTrue(Op->getType()), - IsAnd ? ConstantInt::getFalse(Op->getType()) : V); + if (Value *V = simplifyNestedSelectsUsingImpliedCond(SI, Op, IsAnd, DL)) { + Instruction *MDFrom = nullptr; + if (!ProfcheckDisableMetadataFixes) + MDFrom = &SI; + return SelectInst::Create( + Op, IsAnd ? V : ConstantInt::getTrue(Op->getType()), + IsAnd ? ConstantInt::getFalse(Op->getType()) : V, "", nullptr, MDFrom); + } return nullptr; } diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index 860f8f7..511bca4 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -3337,7 +3337,7 @@ PHINode *FunctionStackPoisoner::createPHI(IRBuilder<> &IRB, Value *Cond, Value *ValueIfTrue, Instruction *ThenTerm, Value *ValueIfFalse) { - PHINode *PHI = IRB.CreatePHI(IntptrTy, 2); + PHINode *PHI = IRB.CreatePHI(ValueIfTrue->getType(), 2); BasicBlock *CondBlock = cast<Instruction>(Cond)->getParent(); PHI->addIncoming(ValueIfFalse, CondBlock); BasicBlock *ThenBlock = ThenTerm->getParent(); @@ -3360,7 +3360,7 @@ Value *FunctionStackPoisoner::createAllocaForLayout( assert((ClRealignStack & (ClRealignStack - 1)) == 0); uint64_t FrameAlignment = std::max(L.FrameAlignment, uint64_t(ClRealignStack)); Alloca->setAlignment(Align(FrameAlignment)); - return IRB.CreatePointerCast(Alloca, IntptrTy); + return Alloca; } void FunctionStackPoisoner::createDynamicAllocasInitStorage() { @@ -3572,10 +3572,12 @@ void FunctionStackPoisoner::processStaticAllocas() { DoDynamicAlloca &= !HasInlineAsm && !HasReturnsTwiceCall; DoStackMalloc &= !HasInlineAsm && !HasReturnsTwiceCall; + Type *PtrTy = F.getDataLayout().getAllocaPtrType(F.getContext()); Value *StaticAlloca = DoDynamicAlloca ? nullptr : createAllocaForLayout(IRB, L, false); - Value *FakeStack; + Value *FakeStackPtr; + Value *FakeStackInt; Value *LocalStackBase; Value *LocalStackBaseAlloca; uint8_t DIExprFlags = DIExpression::ApplyOffset; @@ -3603,20 +3605,21 @@ void FunctionStackPoisoner::processStaticAllocas() { RTCI.createRuntimeCall(IRBIf, AsanStackMallocFunc[StackMallocIdx], ConstantInt::get(IntptrTy, LocalStackSize)); IRB.SetInsertPoint(InsBefore); - FakeStack = createPHI(IRB, UseAfterReturnIsEnabled, FakeStackValue, Term, - ConstantInt::get(IntptrTy, 0)); + FakeStackInt = createPHI(IRB, UseAfterReturnIsEnabled, FakeStackValue, + Term, ConstantInt::get(IntptrTy, 0)); } else { // assert(ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode:Always) // void *FakeStack = __asan_stack_malloc_N(LocalStackSize); // void *LocalStackBase = (FakeStack) ? FakeStack : // alloca(LocalStackSize); StackMallocIdx = StackMallocSizeClass(LocalStackSize); - FakeStack = + FakeStackInt = RTCI.createRuntimeCall(IRB, AsanStackMallocFunc[StackMallocIdx], ConstantInt::get(IntptrTy, LocalStackSize)); } + FakeStackPtr = IRB.CreateIntToPtr(FakeStackInt, PtrTy); Value *NoFakeStack = - IRB.CreateICmpEQ(FakeStack, Constant::getNullValue(IntptrTy)); + IRB.CreateICmpEQ(FakeStackInt, Constant::getNullValue(IntptrTy)); Instruction *Term = SplitBlockAndInsertIfThen(NoFakeStack, InsBefore, false); IRBuilder<> IRBIf(Term); @@ -3624,67 +3627,53 @@ void FunctionStackPoisoner::processStaticAllocas() { DoDynamicAlloca ? createAllocaForLayout(IRBIf, L, true) : StaticAlloca; IRB.SetInsertPoint(InsBefore); - LocalStackBase = createPHI(IRB, NoFakeStack, AllocaValue, Term, FakeStack); + LocalStackBase = + createPHI(IRB, NoFakeStack, AllocaValue, Term, FakeStackPtr); IRB.CreateStore(LocalStackBase, LocalStackBaseAlloca); DIExprFlags |= DIExpression::DerefBefore; } else { // void *FakeStack = nullptr; // void *LocalStackBase = alloca(LocalStackSize); - FakeStack = ConstantInt::get(IntptrTy, 0); + FakeStackInt = Constant::getNullValue(IntptrTy); + FakeStackPtr = Constant::getNullValue(PtrTy); LocalStackBase = DoDynamicAlloca ? createAllocaForLayout(IRB, L, true) : StaticAlloca; LocalStackBaseAlloca = LocalStackBase; } - // It shouldn't matter whether we pass an `alloca` or a `ptrtoint` as the - // dbg.declare address opereand, but passing a `ptrtoint` seems to confuse - // later passes and can result in dropped variable coverage in debug info. - Value *LocalStackBaseAllocaPtr = - isa<PtrToIntInst>(LocalStackBaseAlloca) - ? cast<PtrToIntInst>(LocalStackBaseAlloca)->getPointerOperand() - : LocalStackBaseAlloca; - assert(isa<AllocaInst>(LocalStackBaseAllocaPtr) && - "Variable descriptions relative to ASan stack base will be dropped"); - // Replace Alloca instructions with base+offset. SmallVector<Value *> NewAllocaPtrs; for (const auto &Desc : SVD) { AllocaInst *AI = Desc.AI; - replaceDbgDeclare(AI, LocalStackBaseAllocaPtr, DIB, DIExprFlags, - Desc.Offset); - Value *NewAllocaPtr = IRB.CreateIntToPtr( - IRB.CreateAdd(LocalStackBase, ConstantInt::get(IntptrTy, Desc.Offset)), - AI->getType()); + replaceDbgDeclare(AI, LocalStackBaseAlloca, DIB, DIExprFlags, Desc.Offset); + Value *NewAllocaPtr = IRB.CreatePtrAdd( + LocalStackBase, ConstantInt::get(IntptrTy, Desc.Offset)); AI->replaceAllUsesWith(NewAllocaPtr); NewAllocaPtrs.push_back(NewAllocaPtr); } // The left-most redzone has enough space for at least 4 pointers. // Write the Magic value to redzone[0]. - Value *BasePlus0 = IRB.CreateIntToPtr(LocalStackBase, IntptrPtrTy); IRB.CreateStore(ConstantInt::get(IntptrTy, kCurrentStackFrameMagic), - BasePlus0); + LocalStackBase); // Write the frame description constant to redzone[1]. - Value *BasePlus1 = IRB.CreateIntToPtr( - IRB.CreateAdd(LocalStackBase, - ConstantInt::get(IntptrTy, ASan.LongSize / 8)), - IntptrPtrTy); + Value *BasePlus1 = IRB.CreatePtrAdd( + LocalStackBase, ConstantInt::get(IntptrTy, ASan.LongSize / 8)); GlobalVariable *StackDescriptionGlobal = createPrivateGlobalForString(*F.getParent(), DescriptionString, /*AllowMerging*/ true, genName("stack")); Value *Description = IRB.CreatePointerCast(StackDescriptionGlobal, IntptrTy); IRB.CreateStore(Description, BasePlus1); // Write the PC to redzone[2]. - Value *BasePlus2 = IRB.CreateIntToPtr( - IRB.CreateAdd(LocalStackBase, - ConstantInt::get(IntptrTy, 2 * ASan.LongSize / 8)), - IntptrPtrTy); + Value *BasePlus2 = IRB.CreatePtrAdd( + LocalStackBase, ConstantInt::get(IntptrTy, 2 * ASan.LongSize / 8)); IRB.CreateStore(IRB.CreatePointerCast(&F, IntptrTy), BasePlus2); const auto &ShadowAfterScope = GetShadowBytesAfterScope(SVD, L); // Poison the stack red zones at the entry. - Value *ShadowBase = ASan.memToShadow(LocalStackBase, IRB); + Value *ShadowBase = + ASan.memToShadow(IRB.CreatePtrToInt(LocalStackBase, IntptrTy), IRB); // As mask we must use most poisoned case: red zones and after scope. // As bytes we can use either the same or just red zones only. copyToShadow(ShadowAfterScope, ShadowAfterScope, IRB, ShadowBase); @@ -3723,7 +3712,7 @@ void FunctionStackPoisoner::processStaticAllocas() { IRBuilder<> IRBRet(Ret); // Mark the current frame as retired. IRBRet.CreateStore(ConstantInt::get(IntptrTy, kRetiredStackFrameMagic), - BasePlus0); + LocalStackBase); if (DoStackMalloc) { assert(StackMallocIdx >= 0); // if FakeStack != 0 // LocalStackBase == FakeStack @@ -3737,7 +3726,7 @@ void FunctionStackPoisoner::processStaticAllocas() { // else // <This is not a fake stack; unpoison the redzones> Value *Cmp = - IRBRet.CreateICmpNE(FakeStack, Constant::getNullValue(IntptrTy)); + IRBRet.CreateICmpNE(FakeStackInt, Constant::getNullValue(IntptrTy)); Instruction *ThenTerm, *ElseTerm; SplitBlockAndInsertIfThenElse(Cmp, Ret, &ThenTerm, &ElseTerm); @@ -3748,11 +3737,10 @@ void FunctionStackPoisoner::processStaticAllocas() { kAsanStackUseAfterReturnMagic); copyToShadow(ShadowAfterReturn, ShadowAfterReturn, IRBPoison, ShadowBase); - Value *SavedFlagPtrPtr = IRBPoison.CreateAdd( - FakeStack, + Value *SavedFlagPtrPtr = IRBPoison.CreatePtrAdd( + FakeStackPtr, ConstantInt::get(IntptrTy, ClassSize - ASan.LongSize / 8)); - Value *SavedFlagPtr = IRBPoison.CreateLoad( - IntptrTy, IRBPoison.CreateIntToPtr(SavedFlagPtrPtr, IntptrPtrTy)); + Value *SavedFlagPtr = IRBPoison.CreateLoad(IntptrTy, SavedFlagPtrPtr); IRBPoison.CreateStore( Constant::getNullValue(IRBPoison.getInt8Ty()), IRBPoison.CreateIntToPtr(SavedFlagPtr, IRBPoison.getPtrTy())); @@ -3760,7 +3748,7 @@ void FunctionStackPoisoner::processStaticAllocas() { // For larger frames call __asan_stack_free_*. RTCI.createRuntimeCall( IRBPoison, AsanStackFreeFunc[StackMallocIdx], - {FakeStack, ConstantInt::get(IntptrTy, LocalStackSize)}); + {FakeStackInt, ConstantInt::get(IntptrTy, LocalStackSize)}); } IRBuilder<> IRBElse(ElseTerm); diff --git a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp index ff5f390..66e45ec 100644 --- a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp +++ b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp @@ -266,8 +266,7 @@ void DFAJumpThreading::unfold(DomTreeUpdater *DTU, LoopInfo *LI, if (!ProfcheckDisableMetadataFixes) BI->setMetadata(LLVMContext::MD_prof, SI->getMetadata(LLVMContext::MD_prof)); - DTU->applyUpdates({{DominatorTree::Insert, StartBlock, EndBlock}, - {DominatorTree::Insert, StartBlock, NewBlock}}); + DTU->applyUpdates({{DominatorTree::Insert, StartBlock, NewBlock}}); } else { BasicBlock *EndBlock = SIUse->getParent(); BasicBlock *NewBlockT = BasicBlock::Create( @@ -1479,10 +1478,13 @@ bool DFAJumpThreading::run(Function &F) { DTU->flush(); #ifdef EXPENSIVE_CHECKS - assert(DTU->getDomTree().verify(DominatorTree::VerificationLevel::Full)); verifyFunction(F, &dbgs()); #endif + if (MadeChanges && VerifyDomInfo) + assert(DTU->getDomTree().verify(DominatorTree::VerificationLevel::Full) && + "Failed to maintain validity of domtree!"); + return MadeChanges; } diff --git a/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp b/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp index 213d0f3..1335665 100644 --- a/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp +++ b/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp @@ -39,10 +39,11 @@ public: private: AliasAnalysis *AA; }; +} // namespace /// iterativelyFlattenCFG - Call FlattenCFG on all the blocks in the function, /// iterating until no more changes are made. -bool iterativelyFlattenCFG(Function &F, AliasAnalysis *AA) { +static bool iterativelyFlattenCFG(Function &F, AliasAnalysis *AA) { bool Changed = false; bool LocalChange = true; @@ -67,7 +68,6 @@ bool iterativelyFlattenCFG(Function &F, AliasAnalysis *AA) { } return Changed; } -} // namespace char FlattenCFGLegacyPass::ID = 0; diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index b9d332b..d2f09e9 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -118,9 +118,13 @@ STATISTIC( STATISTIC(NumDeleted, "Number of instructions deleted"); STATISTIC(NumVectorized, "Number of vectorized aggregates"); +namespace llvm { /// Disable running mem2reg during SROA in order to test or debug SROA. static cl::opt<bool> SROASkipMem2Reg("sroa-skip-mem2reg", cl::init(false), cl::Hidden); +extern cl::opt<bool> ProfcheckDisableMetadataFixes; +} // namespace llvm + namespace { class AllocaSliceRewriter; @@ -1777,7 +1781,8 @@ static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI, } Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL, - LI.getName() + ".sroa.speculated"); + LI.getName() + ".sroa.speculated", + ProfcheckDisableMetadataFixes ? nullptr : &SI); LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n"); LI.replaceAllUsesWith(V); @@ -4360,10 +4365,13 @@ private: }; Value *Cond, *True, *False; + Instruction *MDFrom = nullptr; if (auto *SI = dyn_cast<SelectInst>(Sel)) { Cond = SI->getCondition(); True = SI->getTrueValue(); False = SI->getFalseValue(); + if (!ProfcheckDisableMetadataFixes) + MDFrom = SI; } else { Cond = Sel->getOperand(0); True = ConstantInt::get(Sel->getType(), 1); @@ -4383,8 +4391,12 @@ private: IRB.CreateGEP(Ty, FalseOps[0], ArrayRef(FalseOps).drop_front(), False->getName() + ".sroa.gep", NW); - Value *NSel = - IRB.CreateSelect(Cond, NTrue, NFalse, Sel->getName() + ".sroa.sel"); + Value *NSel = MDFrom + ? IRB.CreateSelect(Cond, NTrue, NFalse, + Sel->getName() + ".sroa.sel", MDFrom) + : IRB.CreateSelectWithUnknownProfile( + Cond, NTrue, NFalse, DEBUG_TYPE, + Sel->getName() + ".sroa.sel"); Visited.erase(&GEPI); GEPI.replaceAllUsesWith(NSel); GEPI.eraseFromParent(); diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp index 2ee91a9..0f3978f 100644 --- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp +++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp @@ -47,6 +47,7 @@ #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/SSAUpdater.h" +#include "llvm/Transforms/Utils/SSAUpdaterBulk.h" #include <cassert> #include <utility> @@ -321,7 +322,7 @@ class StructurizeCFG { void collectInfos(); - void insertConditions(bool Loops); + void insertConditions(bool Loops, SSAUpdaterBulk &PhiInserter); void simplifyConditions(); @@ -671,10 +672,9 @@ void StructurizeCFG::collectInfos() { } /// Insert the missing branch conditions -void StructurizeCFG::insertConditions(bool Loops) { +void StructurizeCFG::insertConditions(bool Loops, SSAUpdaterBulk &PhiInserter) { BranchVector &Conds = Loops ? LoopConds : Conditions; Value *Default = Loops ? BoolTrue : BoolFalse; - SSAUpdater PhiInserter; for (BranchInst *Term : Conds) { assert(Term->isConditional()); @@ -683,8 +683,9 @@ void StructurizeCFG::insertConditions(bool Loops) { BasicBlock *SuccTrue = Term->getSuccessor(0); BasicBlock *SuccFalse = Term->getSuccessor(1); - PhiInserter.Initialize(Boolean, ""); - PhiInserter.AddAvailableValue(Loops ? SuccFalse : Parent, Default); + unsigned Variable = PhiInserter.AddVariable("", Boolean); + PhiInserter.AddAvailableValue(Variable, Loops ? SuccFalse : Parent, + Default); BBPredicates &Preds = Loops ? LoopPreds[SuccFalse] : Predicates[SuccTrue]; @@ -697,7 +698,7 @@ void StructurizeCFG::insertConditions(bool Loops) { ParentInfo = PI; break; } - PhiInserter.AddAvailableValue(BB, PI.Pred); + PhiInserter.AddAvailableValue(Variable, BB, PI.Pred); Dominator.addAndRememberBlock(BB); } @@ -706,9 +707,9 @@ void StructurizeCFG::insertConditions(bool Loops) { CondBranchWeights::setMetadata(*Term, ParentInfo.Weights); } else { if (!Dominator.resultIsRememberedBlock()) - PhiInserter.AddAvailableValue(Dominator.result(), Default); + PhiInserter.AddAvailableValue(Variable, Dominator.result(), Default); - Term->setCondition(PhiInserter.GetValueInMiddleOfBlock(Parent)); + PhiInserter.AddUse(Variable, &Term->getOperandUse(0)); } } } @@ -1414,8 +1415,12 @@ bool StructurizeCFG::run(Region *R, DominatorTree *DT, orderNodes(); collectInfos(); createFlow(); - insertConditions(false); - insertConditions(true); + + SSAUpdaterBulk PhiInserter; + insertConditions(false, PhiInserter); + insertConditions(true, PhiInserter); + PhiInserter.RewriteAndOptimizeAllUses(*DT); + setPhiValues(); simplifyHoistedPhis(); simplifyConditions(); diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp index 3ce569f..32924e7 100644 --- a/llvm/lib/Transforms/Utils/CloneFunction.cpp +++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp @@ -826,7 +826,7 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc, BasicBlock::iterator I = NewBB->begin(); for (; (PN = dyn_cast<PHINode>(I)); ++I) { for (const auto &[Pred, Count] : PredCount) { - for (unsigned _ : llvm::seq<unsigned>(Count)) + for ([[maybe_unused]] unsigned _ : llvm::seq<unsigned>(Count)) PN->removeIncomingValue(Pred, false); } } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 8ca3bed..23f5623 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -24,12 +24,9 @@ #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H #define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H -#include "VPlanAnalysis.h" #include "VPlanValue.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/ilist.h" @@ -41,10 +38,11 @@ #include "llvm/IR/Operator.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/InstructionCost.h" -#include <algorithm> #include <cassert> #include <cstddef> +#include <functional> #include <string> +#include <utility> namespace llvm { @@ -346,13 +344,6 @@ public: /// Return the cost of the block. virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx) = 0; - /// Return true if it is legal to hoist instructions into this block. - bool isLegalToHoistInto() { - // There are currently no constraints that prevent an instruction to be - // hoisted into a VPBlockBase. - return true; - } - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void printAsOperand(raw_ostream &OS, bool PrintType = false) const { OS << getName(); |