aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms')
-rw-r--r--llvm/lib/Transforms/IPO/AttributorAttributes.cpp16
-rw-r--r--llvm/lib/Transforms/IPO/OpenMPOpt.cpp2
-rw-r--r--llvm/lib/Transforms/InstCombine/InstCombineInternal.h2
-rw-r--r--llvm/lib/Transforms/InstCombine/InstructionCombining.cpp20
-rw-r--r--llvm/lib/Transforms/Utils/SimplifyCFG.cpp27
-rw-r--r--llvm/lib/Transforms/Vectorize/LoopVectorize.cpp17
-rw-r--r--llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp28
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlan.h10
8 files changed, 81 insertions, 41 deletions
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 6d16599..5048561 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -1044,15 +1044,13 @@ struct AAPointerInfoImpl
return AAPointerInfo::manifest(A);
}
- virtual const_bin_iterator begin() const override { return State::begin(); }
- virtual const_bin_iterator end() const override { return State::end(); }
- virtual int64_t numOffsetBins() const override {
- return State::numOffsetBins();
- }
- virtual bool reachesReturn() const override {
+ const_bin_iterator begin() const override { return State::begin(); }
+ const_bin_iterator end() const override { return State::end(); }
+ int64_t numOffsetBins() const override { return State::numOffsetBins(); }
+ bool reachesReturn() const override {
return !ReturnedOffsets.isUnassigned();
}
- virtual void addReturnedOffsetsTo(OffsetInfo &OI) const override {
+ void addReturnedOffsetsTo(OffsetInfo &OI) const override {
if (ReturnedOffsets.isUnknown()) {
OI.setUnknown();
return;
@@ -6653,7 +6651,7 @@ struct AAHeapToStackFunction final : public AAHeapToStack {
AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
: AAHeapToStack(IRP, A) {}
- ~AAHeapToStackFunction() {
+ ~AAHeapToStackFunction() override {
// Ensure we call the destructor so we release any memory allocated in the
// sets.
for (auto &It : AllocationInfos)
@@ -8374,7 +8372,7 @@ struct AAMemoryLocationImpl : public AAMemoryLocation {
AccessKind2Accesses.fill(nullptr);
}
- ~AAMemoryLocationImpl() {
+ ~AAMemoryLocationImpl() override {
// The AccessSets are allocated via a BumpPtrAllocator, we call
// the destructor manually.
for (AccessSet *AS : AccessKind2Accesses)
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 5e2247f..d7eb745 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -2693,7 +2693,7 @@ struct AAExecutionDomainFunction : public AAExecutionDomain {
AAExecutionDomainFunction(const IRPosition &IRP, Attributor &A)
: AAExecutionDomain(IRP, A) {}
- ~AAExecutionDomainFunction() { delete RPOT; }
+ ~AAExecutionDomainFunction() override { delete RPOT; }
void initialize(Attributor &A) override {
Function *F = getAnchorScope();
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index ede73f8..9c75d9a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -72,7 +72,7 @@ public:
: InstCombiner(Worklist, Builder, F, AA, AC, TLI, TTI, DT, ORE, BFI, BPI,
PSI, DL, RPOT) {}
- virtual ~InstCombinerImpl() = default;
+ ~InstCombinerImpl() override = default;
/// Perform early cleanup and prepare the InstCombine worklist.
bool prepareWorklist(Function &F);
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 9c8de45..67f837c 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -3358,21 +3358,21 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) {
if (TyAllocSize == 1) {
// Canonicalize (gep i8* X, (ptrtoint Y)-(ptrtoint X)) to (bitcast Y),
- // but only if the result pointer is only used as if it were an integer,
- // or both point to the same underlying object (otherwise provenance is
- // not necessarily retained).
+ // but only if the result pointer is only used as if it were an integer.
+ // (The case where the underlying object is the same is handled by
+ // InstSimplify.)
Value *X = GEP.getPointerOperand();
Value *Y;
- if (match(GEP.getOperand(1),
- m_Sub(m_PtrToInt(m_Value(Y)), m_PtrToInt(m_Specific(X)))) &&
+ if (match(GEP.getOperand(1), m_Sub(m_PtrToIntOrAddr(m_Value(Y)),
+ m_PtrToIntOrAddr(m_Specific(X)))) &&
GEPType == Y->getType()) {
- bool HasSameUnderlyingObject =
- getUnderlyingObject(X) == getUnderlyingObject(Y);
+ bool HasNonAddressBits =
+ DL.getAddressSizeInBits(AS) != DL.getPointerSizeInBits(AS);
bool Changed = false;
GEP.replaceUsesWithIf(Y, [&](Use &U) {
- bool ShouldReplace = HasSameUnderlyingObject ||
- isa<ICmpInst>(U.getUser()) ||
- isa<PtrToIntInst>(U.getUser());
+ bool ShouldReplace = isa<PtrToAddrInst>(U.getUser()) ||
+ (!HasNonAddressBits &&
+ isa<ICmpInst, PtrToIntInst>(U.getUser()));
Changed |= ShouldReplace;
return ShouldReplace;
});
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index d831c27..c537be5c 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -7551,6 +7551,7 @@ static bool reduceSwitchRange(SwitchInst *SI, IRBuilder<> &Builder,
/// log2(C)-indexed value table (instead of traditionally emitting a load of the
/// address of the jump target, and indirectly jump to it).
static bool simplifySwitchOfPowersOfTwo(SwitchInst *SI, IRBuilder<> &Builder,
+ DomTreeUpdater *DTU,
const DataLayout &DL,
const TargetTransformInfo &TTI) {
Value *Condition = SI->getCondition();
@@ -7573,12 +7574,6 @@ static bool simplifySwitchOfPowersOfTwo(SwitchInst *SI, IRBuilder<> &Builder,
if (SI->getNumCases() < 4)
return false;
- // We perform this optimization only for switches with
- // unreachable default case.
- // This assumtion will save us from checking if `Condition` is a power of two.
- if (!SI->defaultDestUnreachable())
- return false;
-
// Check that switch cases are powers of two.
SmallVector<uint64_t, 4> Values;
for (const auto &Case : SI->cases()) {
@@ -7598,6 +7593,24 @@ static bool simplifySwitchOfPowersOfTwo(SwitchInst *SI, IRBuilder<> &Builder,
Builder.SetInsertPoint(SI);
+ if (!SI->defaultDestUnreachable()) {
+ // Let non-power-of-two inputs jump to the default case, when the latter is
+ // reachable.
+ auto *PopC = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, Condition);
+ auto *IsPow2 = Builder.CreateICmpEQ(PopC, ConstantInt::get(CondTy, 1));
+
+ auto *OrigBB = SI->getParent();
+ auto *DefaultCaseBB = SI->getDefaultDest();
+ BasicBlock *SplitBB = SplitBlock(OrigBB, SI, DTU);
+ auto It = OrigBB->getTerminator()->getIterator();
+ BranchInst::Create(SplitBB, DefaultCaseBB, IsPow2, It);
+ It->eraseFromParent();
+
+ addPredecessorToBlock(DefaultCaseBB, OrigBB, SplitBB);
+ if (DTU)
+ DTU->applyUpdates({{DominatorTree::Insert, OrigBB, DefaultCaseBB}});
+ }
+
// Replace each case with its trailing zeros number.
for (auto &Case : SI->cases()) {
auto *OrigValue = Case.getCaseValue();
@@ -7953,7 +7966,7 @@ bool SimplifyCFGOpt::simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) {
Options.ConvertSwitchToLookupTable))
return requestResimplify();
- if (simplifySwitchOfPowersOfTwo(SI, Builder, DL, TTI))
+ if (simplifySwitchOfPowersOfTwo(SI, Builder, DTU, DL, TTI))
return requestResimplify();
if (reduceSwitchRange(SI, Builder, DL, TTI))
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3356516..facb0fa 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4378,8 +4378,21 @@ VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
const SCEV *TC =
vputils::getSCEVExprForVPValue(getPlanFor(MainLoopVF).getTripCount(), SE);
assert(!isa<SCEVCouldNotCompute>(TC) && "Trip count SCEV must be computable");
- RemainingIterations =
- SE.getURemExpr(TC, SE.getElementCount(TCType, MainLoopVF * IC));
+ const SCEV *KnownMinTC;
+ bool ScalableTC = match(TC, m_scev_c_Mul(m_SCEV(KnownMinTC), m_SCEVVScale()));
+ // Use versions of TC and VF in which both are either scalable or fixed.
+ if (ScalableTC == MainLoopVF.isScalable())
+ RemainingIterations =
+ SE.getURemExpr(TC, SE.getElementCount(TCType, MainLoopVF * IC));
+ else if (ScalableTC) {
+ const SCEV *EstimatedTC = SE.getMulExpr(
+ KnownMinTC,
+ SE.getConstant(TCType, CM.getVScaleForTuning().value_or(1)));
+ RemainingIterations = SE.getURemExpr(
+ EstimatedTC, SE.getElementCount(TCType, MainLoopVF * IC));
+ } else
+ RemainingIterations =
+ SE.getURemExpr(TC, SE.getElementCount(TCType, EstimatedRuntimeVF * IC));
// No iterations left to process in the epilogue.
if (RemainingIterations->isZero())
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index cdb9e7e..4fcaf6d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -17641,12 +17641,28 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
[](Value *V) {
return !isa<GetElementPtrInst>(V) && isa<Instruction>(V);
})) ||
- all_of(E->Scalars, [&](Value *V) {
- return isa<PoisonValue>(V) ||
- (E->Idx == 0 && isa<InsertElementInst>(V)) ||
- E->isCopyableElement(V) ||
- (!isVectorLikeInstWithConstOps(V) && isUsedOutsideBlock(V));
- }))
+ (all_of(E->Scalars,
+ [&](Value *V) {
+ return isa<PoisonValue>(V) ||
+ (E->Idx == 0 && isa<InsertElementInst>(V)) ||
+ E->isCopyableElement(V) ||
+ (!isVectorLikeInstWithConstOps(V) &&
+ isUsedOutsideBlock(V));
+ }) &&
+ (!E->doesNotNeedToSchedule() ||
+ any_of(E->Scalars,
+ [&](Value *V) {
+ if (!isa<Instruction>(V) ||
+ (E->hasCopyableElements() && E->isCopyableElement(V)))
+ return false;
+ return !areAllOperandsNonInsts(V);
+ }) ||
+ none_of(E->Scalars, [&](Value *V) {
+ if (!isa<Instruction>(V) ||
+ (E->hasCopyableElements() && E->isCopyableElement(V)))
+ return false;
+ return MustGather.contains(V);
+ }))))
Res = FindLastInst();
else
Res = FindFirstInst();
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 2591df8..5b9f005 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -398,7 +398,7 @@ public:
DebugLoc DL = DebugLoc::getUnknown())
: VPDef(SC), VPUser(Operands), DL(DL) {}
- virtual ~VPRecipeBase() = default;
+ ~VPRecipeBase() override = default;
/// Clone the current recipe.
virtual VPRecipeBase *clone() = 0;
@@ -576,7 +576,7 @@ public:
return R && classof(R);
}
- virtual VPSingleDefRecipe *clone() override = 0;
+ VPSingleDefRecipe *clone() override = 0;
/// Returns the underlying instruction.
Instruction *getUnderlyingInstr() {
@@ -907,7 +907,7 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
return R && classof(R);
}
- virtual VPRecipeWithIRFlags *clone() override = 0;
+ VPRecipeWithIRFlags *clone() override = 0;
static inline bool classof(const VPSingleDefRecipe *U) {
auto *R = dyn_cast<VPRecipeBase>(U);
@@ -2068,7 +2068,7 @@ public:
return classof(static_cast<const VPRecipeBase *>(R));
}
- virtual void execute(VPTransformState &State) override = 0;
+ void execute(VPTransformState &State) override = 0;
/// Returns the step value of the induction.
VPValue *getStepValue() { return getOperand(1); }
@@ -2557,7 +2557,7 @@ public:
VPCostContext &Ctx) const override;
/// Returns true if the recipe only uses the first lane of operand \p Op.
- virtual bool onlyFirstLaneUsed(const VPValue *Op) const override = 0;
+ bool onlyFirstLaneUsed(const VPValue *Op) const override = 0;
/// Returns the number of stored operands of this interleave group. Returns 0
/// for load interleave groups.
generated by cgit v1.1 at 2025-10-27 10:48:45 +0000