diff options
author | Paul Kirth <paulkirth@google.com> | 2024-06-10 23:06:06 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-06-11 08:06:06 +0200 |
commit | 607afa0b6375e4837fef298a798f5534e783d777 (patch) | |
tree | bc96cda2dc26b207d70030af09cba3f37d402aad | |
parent | 41c650e8208f7804eb5ecd8749d6b31b6e518bb7 (diff) | |
download | llvm-607afa0b6375e4837fef298a798f5534e783d777.zip llvm-607afa0b6375e4837fef298a798f5534e783d777.tar.gz llvm-607afa0b6375e4837fef298a798f5534e783d777.tar.bz2 |
Revert "[llvm][IR] Extend BranchWeightMetadata to track provenance of weights" (#95060)
Reverts llvm/llvm-project#86609
This change causes compile-time regressions for stage2 builds
(https://llvm-compile-time-tracker.com/compare.php?from=3254f31a66263ea9647c9547f1531c3123444fcd&to=c5978f1eb5eeca8610b9dfce1fcbf1f473911cd8&stat=instructions:u).
It also introduced unintended changes to `.text` which should be
addressed before relanding.
30 files changed, 88 insertions, 169 deletions
diff --git a/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp b/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp index 81d9334..fb236ae 100644 --- a/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp +++ b/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp @@ -221,5 +221,5 @@ void tu2(int &i) { } } -// CHECK: [[BW_LIKELY]] = !{!"branch_weights", !"expected", i32 2000, i32 1} -// CHECK: [[BW_UNLIKELY]] = !{!"branch_weights", !"expected", i32 1, i32 2000} +// CHECK: [[BW_LIKELY]] = !{!"branch_weights", i32 2000, i32 1} +// CHECK: [[BW_UNLIKELY]] = !{!"branch_weights", i32 1, i32 2000} diff --git a/llvm/docs/BranchWeightMetadata.rst b/llvm/docs/BranchWeightMetadata.rst index 6220475..522f37c 100644 --- a/llvm/docs/BranchWeightMetadata.rst +++ b/llvm/docs/BranchWeightMetadata.rst @@ -28,14 +28,11 @@ Supported Instructions Metadata is only assigned to the conditional branches. There are two extra operands for the true and the false branch. -We optionally track if the metadata was added by ``__builtin_expect`` or -``__builtin_expect_with_probability`` with an optional field ``!"expected"``. .. code-block:: none !0 = !{ !"branch_weights", - [ !"expected", ] i32 <TRUE_BRANCH_WEIGHT>, i32 <FALSE_BRANCH_WEIGHT> } @@ -50,7 +47,6 @@ is always case #0). !0 = !{ !"branch_weights", - [ !"expected", ] i32 <DEFAULT_BRANCH_WEIGHT> [ , i32 <CASE_BRANCH_WEIGHT> ... ] } @@ -64,7 +60,6 @@ Branch weights are assigned to every destination. !0 = !{ !"branch_weights", - [ !"expected", ] i32 <LABEL_BRANCH_WEIGHT> [ , i32 <LABEL_BRANCH_WEIGHT> ... ] } @@ -80,7 +75,6 @@ block and entry counts which may not be accurate with sampling. !0 = !{ !"branch_weights", - [ !"expected", ] i32 <CALL_BRANCH_WEIGHT> } @@ -101,7 +95,6 @@ is used. !0 = !{ !"branch_weights", - [ !"expected", ] i32 <INVOKE_NORMAL_WEIGHT> [ , i32 <INVOKE_UNWIND_WEIGHT> ] } diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h index e02ec8f..3265589 100644 --- a/llvm/include/llvm/IR/MDBuilder.h +++ b/llvm/include/llvm/IR/MDBuilder.h @@ -59,11 +59,7 @@ public: //===------------------------------------------------------------------===// /// Return metadata containing two branch weights. - /// @param TrueWeight the weight of the true branch - /// @param FalseWeight the weight of the false branch - /// @param Do these weights come from __builtin_expect* - MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, - bool IsExpected = false); + MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight); /// Return metadata containing two branch weights, with significant bias /// towards `true` destination. @@ -74,10 +70,7 @@ public: MDNode *createUnlikelyBranchWeights(); /// Return metadata containing a number of branch weights. - /// @param Weights the weights of all the branches - /// @param Do these weights come from __builtin_expect* - MDNode *createBranchWeights(ArrayRef<uint32_t> Weights, - bool IsExpected = false); + MDNode *createBranchWeights(ArrayRef<uint32_t> Weights); /// Return metadata specifying that a branch or switch is unpredictable. MDNode *createUnpredictable(); diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h index 1d7c97d..88fbad4 100644 --- a/llvm/include/llvm/IR/ProfDataUtils.h +++ b/llvm/include/llvm/IR/ProfDataUtils.h @@ -55,17 +55,6 @@ MDNode *getBranchWeightMDNode(const Instruction &I); /// Nullptr otherwise. MDNode *getValidBranchWeightMDNode(const Instruction &I); -/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect* -/// intrinsic -bool hasBranchWeightOrigin(const Instruction &I); - -/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect* -/// intrinsic -bool hasBranchWeightOrigin(const MDNode *ProfileData); - -/// Return the offset to the first branch weight data -unsigned getBranchWeightOffset(const MDNode *ProfileData); - /// Extract branch weights from MD_prof metadata /// /// \param ProfileData A pointer to an MDNode. @@ -122,11 +111,7 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalWeights); /// Create a new `branch_weights` metadata node and add or overwrite /// a `prof` metadata reference to instruction `I`. -/// \param I the Instruction to set branch weights on. -/// \param Weights an array of weights to set on instruction I. -/// \param IsExpected were these weights added from an llvm.expect* intrinsic. -void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights, - bool IsExpected); +void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights); /// Scaling the profile data attached to 'I' using the ratio of S/T. void scaleProfData(Instruction &I, uint64_t S, uint64_t T); diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index 0e01080..339a1f1 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -8866,8 +8866,7 @@ bool CodeGenPrepare::splitBranchCondition(Function &F, ModifyDT &ModifiedDT) { scaleWeights(NewTrueWeight, NewFalseWeight); Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext()) - .createBranchWeights(TrueWeight, FalseWeight, - hasBranchWeightOrigin(*Br1))); + .createBranchWeights(TrueWeight, FalseWeight)); NewTrueWeight = TrueWeight; NewFalseWeight = 2 * FalseWeight; diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp index aec927a..29272e6 100644 --- a/llvm/lib/IR/Instruction.cpp +++ b/llvm/lib/IR/Instruction.cpp @@ -1268,23 +1268,12 @@ Instruction *Instruction::cloneImpl() const { void Instruction::swapProfMetadata() { MDNode *ProfileData = getBranchWeightMDNode(*this); - if (!ProfileData) - return; - unsigned FirstIdx = getBranchWeightOffset(ProfileData); - if (ProfileData->getNumOperands() != 2 + FirstIdx) + if (!ProfileData || ProfileData->getNumOperands() != 3) return; - unsigned SecondIdx = FirstIdx + 1; - SmallVector<Metadata *, 4> Ops; - // If there are more weights past the second, we can't swap them - if (ProfileData->getNumOperands() > SecondIdx + 1) - return; - for (unsigned Idx = 0; Idx < FirstIdx; ++Idx) { - Ops.push_back(ProfileData->getOperand(Idx)); - } - // Switch the order of the weights - Ops.push_back(ProfileData->getOperand(SecondIdx)); - Ops.push_back(ProfileData->getOperand(FirstIdx)); + // The first operand is the name. Fetch them backwards and build a new one. + Metadata *Ops[] = {ProfileData->getOperand(0), ProfileData->getOperand(2), + ProfileData->getOperand(1)}; setMetadata(LLVMContext::MD_prof, MDNode::get(ProfileData->getContext(), Ops)); } diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index de369bd..1213f07 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -5199,11 +5199,7 @@ void SwitchInstProfUpdateWrapper::init() { if (!ProfileData) return; - // FIXME: This check belongs in ProfDataUtils. Its almost equivalent to - // getValidBranchWeightMDNode(), but the need to use llvm_unreachable - // makes them slightly different. - if (ProfileData->getNumOperands() != - SI.getNumSuccessors() + getBranchWeightOffset(ProfileData)) { + if (ProfileData->getNumOperands() != SI.getNumSuccessors() + 1) { llvm_unreachable("number of prof branch_weights metadata operands does " "not correspond to number of succesors"); } diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp index 0000277..bd68db3 100644 --- a/llvm/lib/IR/MDBuilder.cpp +++ b/llvm/lib/IR/MDBuilder.cpp @@ -35,8 +35,8 @@ MDNode *MDBuilder::createFPMath(float Accuracy) { } MDNode *MDBuilder::createBranchWeights(uint32_t TrueWeight, - uint32_t FalseWeight, bool IsExpected) { - return createBranchWeights({TrueWeight, FalseWeight}, IsExpected); + uint32_t FalseWeight) { + return createBranchWeights({TrueWeight, FalseWeight}); } MDNode *MDBuilder::createLikelyBranchWeights() { @@ -49,19 +49,15 @@ MDNode *MDBuilder::createUnlikelyBranchWeights() { return createBranchWeights(1, (1U << 20) - 1); } -MDNode *MDBuilder::createBranchWeights(ArrayRef<uint32_t> Weights, - bool IsExpected) { +MDNode *MDBuilder::createBranchWeights(ArrayRef<uint32_t> Weights) { assert(Weights.size() >= 1 && "Need at least one branch weights!"); - unsigned int Offset = IsExpected ? 2 : 1; - SmallVector<Metadata *, 4> Vals(Weights.size() + Offset); + SmallVector<Metadata *, 4> Vals(Weights.size() + 1); Vals[0] = createString("branch_weights"); - if (IsExpected) - Vals[1] = createString("expected"); Type *Int32Ty = Type::getInt32Ty(Context); for (unsigned i = 0, e = Weights.size(); i != e; ++i) - Vals[i + Offset] = createConstant(ConstantInt::get(Int32Ty, Weights[i])); + Vals[i + 1] = createConstant(ConstantInt::get(Int32Ty, Weights[i])); return MDNode::get(Context, Vals); } diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp index 5f42ce2..b6c9324 100644 --- a/llvm/lib/IR/Metadata.cpp +++ b/llvm/lib/IR/Metadata.cpp @@ -1196,10 +1196,10 @@ MDNode *MDNode::mergeDirectCallProfMetadata(MDNode *A, MDNode *B, StringRef AProfName = AMDS->getString(); StringRef BProfName = BMDS->getString(); if (AProfName == "branch_weights" && BProfName == "branch_weights") { - ConstantInt *AInstrWeight = mdconst::dyn_extract<ConstantInt>( - A->getOperand(getBranchWeightOffset(A))); - ConstantInt *BInstrWeight = mdconst::dyn_extract<ConstantInt>( - B->getOperand(getBranchWeightOffset(B))); + ConstantInt *AInstrWeight = + mdconst::dyn_extract<ConstantInt>(A->getOperand(1)); + ConstantInt *BInstrWeight = + mdconst::dyn_extract<ConstantInt>(B->getOperand(1)); assert(AInstrWeight && BInstrWeight && "verified by LLVM verifier"); return MDNode::get(Ctx, {MDHelper.createString("branch_weights"), diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp index c4b1ed5..51e78dc 100644 --- a/llvm/lib/IR/ProfDataUtils.cpp +++ b/llvm/lib/IR/ProfDataUtils.cpp @@ -40,6 +40,9 @@ namespace { // We maintain some constants here to ensure that we access the branch weights // correctly, and can change the behavior in the future if the layout changes +// The index at which the weights vector starts +constexpr unsigned WeightsIdx = 1; + // the minimum number of operands for MD_prof nodes with branch weights constexpr unsigned MinBWOps = 3; @@ -72,7 +75,6 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData, assert(isBranchWeightMD(ProfileData) && "wrong metadata"); unsigned NOps = ProfileData->getNumOperands(); - unsigned WeightsIdx = getBranchWeightOffset(ProfileData); assert(WeightsIdx < NOps && "Weights Index must be less than NOps."); Weights.resize(NOps - WeightsIdx); @@ -80,8 +82,8 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData, ConstantInt *Weight = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(Idx)); assert(Weight && "Malformed branch_weight in MD_prof node"); - assert(Weight->getValue().getActiveBits() <= (sizeof(T) * 8) && - "Too many bits for MD_prof branch_weight"); + assert(Weight->getValue().getActiveBits() <= 32 && + "Too many bits for uint32_t"); Weights[Idx - WeightsIdx] = Weight->getZExtValue(); } } @@ -121,26 +123,6 @@ bool hasValidBranchWeightMD(const Instruction &I) { return getValidBranchWeightMDNode(I); } -bool hasBranchWeightOrigin(const Instruction &I) { - auto *ProfileData = I.getMetadata(LLVMContext::MD_prof); - return hasBranchWeightOrigin(ProfileData); -} - -bool hasBranchWeightOrigin(const MDNode *ProfileData) { - if (!isBranchWeightMD(ProfileData)) - return false; - auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(1)); - // NOTE: if we ever have more types of branch weight provenance, - // we need to check the string value is "expected". For now, we - // supply a more generic API, and avoid the spurious comparisons. - assert(ProfDataName == nullptr || ProfDataName->getString() == "expected"); - return ProfDataName != nullptr; -} - -unsigned getBranchWeightOffset(const MDNode *ProfileData) { - return hasBranchWeightOrigin(ProfileData) ? 2 : 1; -} - MDNode *getBranchWeightMDNode(const Instruction &I) { auto *ProfileData = I.getMetadata(LLVMContext::MD_prof); if (!isBranchWeightMD(ProfileData)) @@ -150,9 +132,7 @@ MDNode *getBranchWeightMDNode(const Instruction &I) { MDNode *getValidBranchWeightMDNode(const Instruction &I) { auto *ProfileData = getBranchWeightMDNode(I); - auto Offset = getBranchWeightOffset(ProfileData); - if (ProfileData && - ProfileData->getNumOperands() == Offset + I.getNumSuccessors()) + if (ProfileData && ProfileData->getNumOperands() == 1 + I.getNumSuccessors()) return ProfileData; return nullptr; } @@ -211,8 +191,7 @@ bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalVal) { return false; if (ProfDataName->getString() == "branch_weights") { - unsigned Offset = getBranchWeightOffset(ProfileData); - for (unsigned Idx = Offset; Idx < ProfileData->getNumOperands(); ++Idx) { + for (unsigned Idx = 1; Idx < ProfileData->getNumOperands(); Idx++) { auto *V = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(Idx)); assert(V && "Malformed branch_weight in MD_prof node"); TotalVal += V->getValue().getZExtValue(); @@ -233,10 +212,9 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalVal) { return extractProfTotalWeight(I.getMetadata(LLVMContext::MD_prof), TotalVal); } -void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights, - bool IsExpected) { +void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights) { MDBuilder MDB(I.getContext()); - MDNode *BranchWeights = MDB.createBranchWeights(Weights, IsExpected); + MDNode *BranchWeights = MDB.createBranchWeights(Weights); I.setMetadata(LLVMContext::MD_prof, BranchWeights); } diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index fe2253d..e592720 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -104,7 +104,6 @@ #include "llvm/IR/Module.h" #include "llvm/IR/ModuleSlotTracker.h" #include "llvm/IR/PassManager.h" -#include "llvm/IR/ProfDataUtils.h" #include "llvm/IR/Statepoint.h" #include "llvm/IR/Type.h" #include "llvm/IR/Use.h" @@ -4809,10 +4808,8 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) { // Check consistency of !prof branch_weights metadata. if (ProfName == "branch_weights") { - unsigned int Offset = getBranchWeightOffset(MD); if (isa<InvokeInst>(&I)) { - Check(MD->getNumOperands() == (1 + Offset) || - MD->getNumOperands() == (2 + Offset), + Check(MD->getNumOperands() == 2 || MD->getNumOperands() == 3, "Wrong number of InvokeInst branch_weights operands", MD); } else { unsigned ExpectedNumOperands = 0; @@ -4832,10 +4829,10 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) { CheckFailed("!prof branch_weights are not allowed for this instruction", MD); - Check(MD->getNumOperands() == Offset + ExpectedNumOperands, + Check(MD->getNumOperands() == 1 + ExpectedNumOperands, "Wrong number of operands", MD); } - for (unsigned i = Offset; i < MD->getNumOperands(); ++i) { + for (unsigned i = 1; i < MD->getNumOperands(); ++i) { auto &MDO = MD->getOperand(i); Check(MDO, "second operand should not be null", MD); Check(mdconst::dyn_extract<ConstantInt>(MDO), diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp index 7e6a881..92ad4c3 100644 --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -1662,8 +1662,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) { else if (OverwriteExistingWeights) I.setMetadata(LLVMContext::MD_prof, nullptr); } else if (!isa<IntrinsicInst>(&I)) { - setBranchWeights(I, {static_cast<uint32_t>(BlockWeights[BB])}, - /*IsExpected=*/false); + setBranchWeights(I, {static_cast<uint32_t>(BlockWeights[BB])}); } } } else if (OverwriteExistingWeights || ProfileSampleBlockAccurate) { @@ -1674,7 +1673,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) { if (cast<CallBase>(I).isIndirectCall()) { I.setMetadata(LLVMContext::MD_prof, nullptr); } else { - setBranchWeights(I, {uint32_t(0)}, /*IsExpected=*/false); + setBranchWeights(I, {uint32_t(0)}); } } } @@ -1757,7 +1756,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) { if (MaxWeight > 0 && (!TI->extractProfTotalWeight(TempWeight) || OverwriteExistingWeights)) { LLVM_DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n"); - setBranchWeights(*TI, Weights, /*IsExpected=*/false); + setBranchWeights(*TI, Weights); ORE->emit([&]() { return OptimizationRemark(DEBUG_TYPE, "PopularDest", MaxDestInst) << "most popular destination for conditional branches at " diff --git a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp index 731104d..0a3d8d6 100644 --- a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp +++ b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp @@ -1878,7 +1878,7 @@ void CHR::fixupBranchesAndSelects(CHRScope *Scope, static_cast<uint32_t>(CHRBranchBias.scale(1000)), static_cast<uint32_t>(CHRBranchBias.getCompl().scale(1000)), }; - setBranchWeights(*MergedBR, Weights, /*IsExpected=*/false); + setBranchWeights(*MergedBR, Weights); CHR_DEBUG(dbgs() << "CHR branch bias " << Weights[0] << ":" << Weights[1] << "\n"); } diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index 6db76ca..23a7c6a 100644 --- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -259,8 +259,7 @@ CallBase &llvm::pgo::promoteIndirectCall(CallBase &CB, Function *DirectCallee, promoteCallWithIfThenElse(CB, DirectCallee, BranchWeights); if (AttachProfToDirectCall) { - setBranchWeights(NewInst, {static_cast<uint32_t>(Count)}, - /*IsExpected=*/false); + setBranchWeights(NewInst, {static_cast<uint32_t>(Count)}); } using namespace ore; diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index ac6d334..2269c2e 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -1474,8 +1474,7 @@ void PGOUseFunc::populateCoverage(IndexedInstrProfReader *PGOReader) { for (auto *Succ : successors(&BB)) Weights.push_back((Coverage[Succ] || !Coverage[&BB]) ? 1 : 0); if (Weights.size() >= 2) - llvm::setBranchWeights(*BB.getTerminator(), Weights, - /*IsExpected=*/false); + llvm::setBranchWeights(*BB.getTerminator(), Weights); } unsigned NumCorruptCoverage = 0; @@ -2261,7 +2260,7 @@ void llvm::setProfMetadata(Module *M, Instruction *TI, misexpect::checkExpectAnnotations(*TI, Weights, /*IsFrontend=*/false); - setBranchWeights(*TI, Weights, /*IsExpected=*/false); + setBranchWeights(*TI, Weights); if (EmitBranchProbability) { std::string BrCondStr = getBranchCondString(TI); if (BrCondStr.empty()) diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp index b958383..74a8f19 100644 --- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp +++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp @@ -231,7 +231,7 @@ static void updatePredecessorProfileMetadata(PHINode *PN, BasicBlock *BB) { Weights[0] = BP.getCompl().getNumerator(); Weights[1] = BP.getNumerator(); } - setBranchWeights(*PredBr, Weights, hasBranchWeightOrigin(*PredBr)); + setBranchWeights(*PredBr, Weights); } } @@ -2618,7 +2618,7 @@ void JumpThreadingPass::updateBlockFreqAndEdgeWeight(BasicBlock *PredBB, Weights.push_back(Prob.getNumerator()); auto TI = BB->getTerminator(); - setBranchWeights(*TI, Weights, hasBranchWeightOrigin(*TI)); + setBranchWeights(*TI, Weights); } } diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp index 17c5a4e..6f87e4d 100644 --- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp +++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp @@ -102,7 +102,7 @@ static bool handleSwitchExpect(SwitchInst &SI) { misexpect::checkExpectAnnotations(SI, Weights, /*IsFrontend=*/true); SI.setCondition(ArgValue); - setBranchWeights(SI, Weights, /*IsExpected=*/true); + setBranchWeights(SI, Weights); return true; } @@ -262,13 +262,11 @@ static void handlePhiDef(CallInst *Expect) { if (IsOpndComingFromSuccessor(BI->getSuccessor(1))) BI->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(LikelyBranchWeightVal, - UnlikelyBranchWeightVal, - /*IsExpected=*/true)); + UnlikelyBranchWeightVal)); else if (IsOpndComingFromSuccessor(BI->getSuccessor(0))) BI->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(UnlikelyBranchWeightVal, - LikelyBranchWeightVal, - /*IsExpected=*/true)); + LikelyBranchWeightVal)); } } @@ -333,12 +331,12 @@ template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) { SmallVector<uint32_t, 4> ExpectedWeights; if ((ExpectedValue->getZExtValue() == ValueComparedTo) == (Predicate == CmpInst::ICMP_EQ)) { - Node = MDB.createBranchWeights( - LikelyBranchWeightVal, UnlikelyBranchWeightVal, /*IsExpected=*/true); + Node = + MDB.createBranchWeights(LikelyBranchWeightVal, UnlikelyBranchWeightVal); ExpectedWeights = {LikelyBranchWeightVal, UnlikelyBranchWeightVal}; } else { - Node = MDB.createBranchWeights(UnlikelyBranchWeightVal, - LikelyBranchWeightVal, /*IsExpected=*/true); + Node = + MDB.createBranchWeights(UnlikelyBranchWeightVal, LikelyBranchWeightVal); ExpectedWeights = {UnlikelyBranchWeightVal, LikelyBranchWeightVal}; } diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index 1222912..ce0f4c7 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -231,7 +231,7 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions, // Remove weight for this case. std::swap(Weights[Idx + 1], Weights.back()); Weights.pop_back(); - setBranchWeights(*SI, Weights, hasBranchWeightOrigin(MD)); + setBranchWeights(*SI, Weights); } // Remove this entry. BasicBlock *ParentBB = SI->getParent(); diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp index d517ec3..e251693 100644 --- a/llvm/lib/Transforms/Utils/LoopPeel.cpp +++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp @@ -680,7 +680,7 @@ struct WeightInfo { /// To avoid dealing with division rounding we can just multiple both part /// of weights to E and use weight as (F - I * E, E). static void updateBranchWeights(Instruction *Term, WeightInfo &Info) { - setBranchWeights(*Term, Info.Weights, /*IsExpected=*/false); + setBranchWeights(*Term, Info.Weights); for (auto [Idx, SubWeight] : enumerate(Info.SubWeights)) if (SubWeight != 0) // Don't set the probability of taking the edge from latch to loop header @@ -1073,7 +1073,7 @@ bool llvm::peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI, } for (const auto &[Term, Info] : Weights) { - setBranchWeights(*Term, Info.Weights, /*IsExpected=*/false); + setBranchWeights(*Term, Info.Weights); } // Update Metadata for count of peeled off iterations. diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp index 04042e7..3d950b1 100644 --- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp @@ -390,13 +390,13 @@ static void updateBranchWeights(BranchInst &PreHeaderBI, BranchInst &LoopBI, SuccsSwapped ? LoopBackWeight : ExitWeight1, SuccsSwapped ? ExitWeight1 : LoopBackWeight, }; - setBranchWeights(LoopBI, LoopBIWeights, /*IsExpected=*/false); + setBranchWeights(LoopBI, LoopBIWeights); if (HasConditionalPreHeader) { const uint32_t PreHeaderBIWeights[] = { SuccsSwapped ? EnterWeight : ExitWeight0, SuccsSwapped ? ExitWeight0 : EnterWeight, }; - setBranchWeights(PreHeaderBI, PreHeaderBIWeights, /*IsExpected=*/false); + setBranchWeights(PreHeaderBI, PreHeaderBIWeights); } } diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 107c8bb..292739b 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -861,28 +861,26 @@ static bool ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1, // Set branch weights on SwitchInst. This sets the metadata if there is at // least one non-zero weight. -static void setBranchWeights(SwitchInst *SI, ArrayRef<uint32_t> Weights, - bool IsExpected) { +static void setBranchWeights(SwitchInst *SI, ArrayRef<uint32_t> Weights) { // Check that there is at least one non-zero weight. Otherwise, pass // nullptr to setMetadata which will erase the existing metadata. MDNode *N = nullptr; if (llvm::any_of(Weights, [](uint32_t W) { return W != 0; })) - N = MDBuilder(SI->getParent()->getContext()) - .createBranchWeights(Weights, IsExpected); + N = MDBuilder(SI->getParent()->getContext()).createBranchWeights(Weights); SI->setMetadata(LLVMContext::MD_prof, N); } // Similar to the above, but for branch and select instructions that take // exactly 2 weights. static void setBranchWeights(Instruction *I, uint32_t TrueWeight, - uint32_t FalseWeight, bool IsExpected) { + uint32_t FalseWeight) { assert(isa<BranchInst>(I) || isa<SelectInst>(I)); // Check that there is at least one non-zero weight. Otherwise, pass // nullptr to setMetadata which will erase the existing metadata. MDNode *N = nullptr; if (TrueWeight || FalseWeight) N = MDBuilder(I->getParent()->getContext()) - .createBranchWeights(TrueWeight, FalseWeight, IsExpected); + .createBranchWeights(TrueWeight, FalseWeight); I->setMetadata(LLVMContext::MD_prof, N); } @@ -1340,7 +1338,7 @@ bool SimplifyCFGOpt::PerformValueComparisonIntoPredecessorFolding( SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end()); - setBranchWeights(NewSI, MDWeights, /*IsExpected=*/false); + setBranchWeights(NewSI, MDWeights); } EraseTerminatorAndDCECond(PTI); @@ -3833,7 +3831,7 @@ static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI, FitWeights(NewWeights); SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(), NewWeights.end()); - setBranchWeights(PBI, MDWeights[0], MDWeights[1], /*IsExpected=*/false); + setBranchWeights(PBI, MDWeights[0], MDWeights[1]); // TODO: If BB is reachable from all paths through PredBlock, then we // could replace PBI's branch probabilities with BI's. @@ -4570,7 +4568,7 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI, // Halve the weights if any of them cannot fit in an uint32_t FitWeights(NewWeights); - setBranchWeights(PBI, NewWeights[0], NewWeights[1], /*IsExpected=*/false); + setBranchWeights(PBI, NewWeights[0], NewWeights[1]); } // OtherDest may have phi nodes. If so, add an entry from PBI's @@ -4606,8 +4604,7 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI, FitWeights(NewWeights); - setBranchWeights(NV, NewWeights[0], NewWeights[1], - /*IsExpected=*/false); + setBranchWeights(NV, NewWeights[0], NewWeights[1]); } } } @@ -4670,7 +4667,7 @@ bool SimplifyCFGOpt::SimplifyTerminatorOnSelect(Instruction *OldTerm, // Create a conditional branch sharing the condition of the select. BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB); if (TrueWeight != FalseWeight) - setBranchWeights(NewBI, TrueWeight, FalseWeight, /*IsExpected=*/false); + setBranchWeights(NewBI, TrueWeight, FalseWeight); } } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) { // Neither of the selected blocks were successors, so this @@ -5620,7 +5617,7 @@ bool SimplifyCFGOpt::TurnSwitchRangeIntoICmp(SwitchInst *SI, TrueWeight /= 2; FalseWeight /= 2; } - setBranchWeights(NewBI, TrueWeight, FalseWeight, /*IsExpected=*/false); + setBranchWeights(NewBI, TrueWeight, FalseWeight); } } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 1acecf2..c7c19ef 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2145,7 +2145,7 @@ public: BranchInst &BI = *BranchInst::Create(Bypass, LoopVectorPreHeader, Cond); if (AddBranchWeights) - setBranchWeights(BI, SCEVCheckBypassWeights, /*IsExpected=*/false); + setBranchWeights(BI, SCEVCheckBypassWeights); ReplaceInstWithInst(SCEVCheckBlock->getTerminator(), &BI); return SCEVCheckBlock; } @@ -2173,7 +2173,7 @@ public: BranchInst &BI = *BranchInst::Create(Bypass, LoopVectorPreHeader, MemRuntimeCheckCond); if (AddBranchWeights) { - setBranchWeights(BI, MemCheckBypassWeights, /*IsExpected=*/false); + setBranchWeights(BI, MemCheckBypassWeights); } ReplaceInstWithInst(MemCheckBlock->getTerminator(), &BI); MemCheckBlock->getTerminator()->setDebugLoc( @@ -2889,7 +2889,7 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { BranchInst &BI = *BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters); if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) - setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false); + setBranchWeights(BI, MinItersBypassWeights); ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI); LoopBypassBlocks.push_back(TCCheckBlock); } @@ -3128,7 +3128,7 @@ BasicBlock *InnerLoopVectorizer::completeLoopSkeleton() { unsigned TripCount = UF * VF.getKnownMinValue(); assert(TripCount > 0 && "trip count should not be zero"); const uint32_t Weights[] = {1, TripCount - 1}; - setBranchWeights(BI, Weights, /*IsExpected=*/false); + setBranchWeights(BI, Weights); } } @@ -7669,7 +7669,7 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass, BranchInst &BI = *BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters); if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) - setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false); + setBranchWeights(BI, MinItersBypassWeights); ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI); return TCCheckBlock; @@ -7826,7 +7826,7 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck( unsigned EstimatedSkipCount = std::min(MainLoopStep, EpilogueLoopStep); const uint32_t Weights[] = {EstimatedSkipCount, MainLoopStep - EstimatedSkipCount}; - setBranchWeights(BI, Weights, /*IsExpected=*/false); + setBranchWeights(BI, Weights); } ReplaceInstWithInst(Insert->getTerminator(), &BI); diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll b/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll index 8e06cd5..0abca5b 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll @@ -284,7 +284,7 @@ define i32 @test10(i64 %t6) { declare i1 @llvm.expect.i1(i1, i1) nounwind readnone -; CHECK: !0 = !{!"branch_weights", !"expected", i32 2000, i32 1} -; CHECK: !1 = !{!"branch_weights", !"expected", i32 1, i32 2000} -; CHECK: !2 = !{!"branch_weights", !"expected", i32 1, i32 1, i32 2000} -; CHECK: !3 = !{!"branch_weights", !"expected", i32 2000, i32 1, i32 1} +; CHECK: !0 = !{!"branch_weights", i32 2000, i32 1} +; CHECK: !1 = !{!"branch_weights", i32 1, i32 2000} +; CHECK: !2 = !{!"branch_weights", i32 1, i32 1, i32 2000} +; CHECK: !3 = !{!"branch_weights", i32 2000, i32 1, i32 1} diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll b/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll index 4057127..6429355 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll @@ -284,7 +284,7 @@ define i32 @test10(i64 %t6) { declare i1 @llvm.expect.with.probability.i1(i1, i1, double) nounwind readnone -; CHECK: !0 = !{!"branch_weights", !"expected", i32 1717986918, i32 429496731} -; CHECK: !1 = !{!"branch_weights", !"expected", i32 429496731, i32 1717986918} -; CHECK: !2 = !{!"branch_weights", !"expected", i32 214748366, i32 214748366, i32 1717986918} -; CHECK: !3 = !{!"branch_weights", !"expected", i32 1717986918, i32 214748366, i32 214748366} +; CHECK: !0 = !{!"branch_weights", i32 1717986918, i32 429496731} +; CHECK: !1 = !{!"branch_weights", i32 429496731, i32 1717986918} +; CHECK: !2 = !{!"branch_weights", i32 214748366, i32 214748366, i32 1717986918} +; CHECK: !3 = !{!"branch_weights", i32 1717986918, i32 214748366, i32 214748366} diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll b/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll index 458a775..2bcfb1e 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll @@ -99,5 +99,6 @@ attributes #1 = { nounwind readnone } !0 = !{i32 1, !"wchar_size", i32 4} !1 = !{!"clang version 5.0.0 (trunk 304373)"} -; CHECK: [[LIKELY]] = !{!"branch_weights", !"expected", i32 2000, i32 1} -; CHECK: [[UNLIKELY]] = !{!"branch_weights", !"expected", i32 1, i32 2000} +; CHECK: [[LIKELY]] = !{!"branch_weights", i32 2000, i32 1} +; CHECK: [[UNLIKELY]] = !{!"branch_weights", i32 1, i32 2000} + diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll index 9b9d9a7..32ae9b0 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll @@ -352,5 +352,5 @@ declare i64 @llvm.expect.i64(i64, i64) !llvm.ident = !{!0} !0 = !{!"clang version 5.0.0 (trunk 302965)"} -; CHECK: [[WEIGHT]] = !{!"branch_weights", !"expected", i32 2000, i32 1} -; CHECK: [[WEIGHT2]] = !{!"branch_weights", !"expected", i32 1, i32 2000} +; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 2000, i32 1} +; CHECK: [[WEIGHT2]] = !{!"branch_weights", i32 1, i32 2000} diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll index e9a843225..1efa632 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll @@ -99,5 +99,5 @@ declare i64 @llvm.expect.i64(i64, i64) !0 = !{!"clang version 5.0.0 (trunk 302965)"} -; CHECK: [[WEIGHT]] = !{!"branch_weights", !"expected", i32 2000, i32 1} -; CHECK: [[WEIGHT2]] = !{!"branch_weights", !"expected", i32 1, i32 2000} +; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 2000, i32 1} +; CHECK: [[WEIGHT2]] = !{!"branch_weights", i32 1, i32 2000} diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll index 13db2c3..9cbaca8 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll @@ -53,4 +53,4 @@ declare i64 @llvm.expect.i64(i64, i64) !0 = !{!"clang version 5.0.0 (trunk 302965)"} -; CHECK: [[WEIGHT]] = !{!"branch_weights", !"expected", i32 1, i32 2000} +; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 1, i32 2000} diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll index 275731d..2bad663 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll @@ -235,5 +235,5 @@ block5: ret void } -; CHECK: !0 = !{!"branch_weights", !"expected", i32 2147483647, i32 1} -; CHECK: !1 = !{!"branch_weights", !"expected", i32 1, i32 2147483647} +; CHECK: !0 = !{!"branch_weights", i32 2147483647, i32 1} +; CHECK: !1 = !{!"branch_weights", i32 1, i32 2147483647} diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll index a090fe2..55dd28b 100644 --- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll +++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll @@ -293,7 +293,7 @@ declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} ; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} ; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]} -; CHECK: [[PROF4]] = !{!"branch_weights", !"expected", i32 2000, i32 1} +; CHECK: [[PROF4]] = !{!"branch_weights", i32 2000, i32 1} ; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]} ; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]} ;. |