diff options
Diffstat (limited to 'llvm/lib/CodeGen/MachineBlockPlacement.cpp')
-rw-r--r-- | llvm/lib/CodeGen/MachineBlockPlacement.cpp | 145 |
1 files changed, 73 insertions, 72 deletions
diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp index 603c0e9..9d46c14 100644 --- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp +++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp @@ -444,9 +444,9 @@ class MachineBlockPlacement : public MachineFunctionPass { if (UseProfileCount) { auto Count = MBFI->getBlockProfileCount(BB); if (Count) - return *Count; + return BlockFrequency(*Count); else - return 0; + return BlockFrequency(0); } else return MBFI->getBlockFreq(BB); } @@ -795,10 +795,10 @@ bool MachineBlockPlacement::shouldTailDuplicate(MachineBasicBlock *BB) { /// penalty is less than 100% /// TODO(iteratee): Use 64-bit fixed point edge frequencies everywhere. static bool greaterWithBias(BlockFrequency A, BlockFrequency B, - uint64_t EntryFreq) { + BlockFrequency EntryFreq) { BranchProbability ThresholdProb(TailDupPlacementPenalty, 100); BlockFrequency Gain = A - B; - return (Gain / ThresholdProb).getFrequency() >= EntryFreq; + return (Gain / ThresholdProb) >= EntryFreq; } /// Check the edge frequencies to see if tail duplication will increase @@ -843,7 +843,7 @@ bool MachineBlockPlacement::isProfitableToTailDup( auto SuccFreq = MBFI->getBlockFreq(Succ); BlockFrequency P = BBFreq * PProb; BlockFrequency Qout = BBFreq * QProb; - uint64_t EntryFreq = MBFI->getEntryFreq(); + BlockFrequency EntryFreq = MBFI->getEntryFreq(); // If there are no more successors, it is profitable to copy, as it strictly // increases fallthrough. if (SuccSuccs.size() == 0) @@ -1927,7 +1927,7 @@ BlockFrequency MachineBlockPlacement::TopFallThroughFreq( const MachineBasicBlock *Top, const BlockFilterSet &LoopBlockSet) { - BlockFrequency MaxFreq = 0; + BlockFrequency MaxFreq = BlockFrequency(0); for (MachineBasicBlock *Pred : Top->predecessors()) { BlockChain *PredChain = BlockToChain[Pred]; if (!LoopBlockSet.count(Pred) && @@ -1986,7 +1986,7 @@ MachineBlockPlacement::FallThroughGains( const MachineBasicBlock *ExitBB, const BlockFilterSet &LoopBlockSet) { BlockFrequency FallThrough2Top = TopFallThroughFreq(OldTop, LoopBlockSet); - BlockFrequency FallThrough2Exit = 0; + BlockFrequency FallThrough2Exit = BlockFrequency(0); if (ExitBB) FallThrough2Exit = MBFI->getBlockFreq(NewTop) * MBPI->getEdgeProbability(NewTop, ExitBB); @@ -1994,58 +1994,58 @@ MachineBlockPlacement::FallThroughGains( MBPI->getEdgeProbability(NewTop, OldTop); // Find the best Pred of NewTop. - MachineBasicBlock *BestPred = nullptr; - BlockFrequency FallThroughFromPred = 0; - for (MachineBasicBlock *Pred : NewTop->predecessors()) { - if (!LoopBlockSet.count(Pred)) - continue; - BlockChain *PredChain = BlockToChain[Pred]; - if (!PredChain || Pred == *std::prev(PredChain->end())) { - BlockFrequency EdgeFreq = MBFI->getBlockFreq(Pred) * - MBPI->getEdgeProbability(Pred, NewTop); - if (EdgeFreq > FallThroughFromPred) { - FallThroughFromPred = EdgeFreq; - BestPred = Pred; - } - } - } - - // If NewTop is not placed after Pred, another successor can be placed - // after Pred. - BlockFrequency NewFreq = 0; - if (BestPred) { - for (MachineBasicBlock *Succ : BestPred->successors()) { - if ((Succ == NewTop) || (Succ == BestPred) || !LoopBlockSet.count(Succ)) - continue; - if (ComputedEdges.contains(Succ)) - continue; - BlockChain *SuccChain = BlockToChain[Succ]; - if ((SuccChain && (Succ != *SuccChain->begin())) || - (SuccChain == BlockToChain[BestPred])) - continue; - BlockFrequency EdgeFreq = MBFI->getBlockFreq(BestPred) * - MBPI->getEdgeProbability(BestPred, Succ); - if (EdgeFreq > NewFreq) - NewFreq = EdgeFreq; - } - BlockFrequency OrigEdgeFreq = MBFI->getBlockFreq(BestPred) * - MBPI->getEdgeProbability(BestPred, NewTop); - if (NewFreq > OrigEdgeFreq) { - // If NewTop is not the best successor of Pred, then Pred doesn't - // fallthrough to NewTop. So there is no FallThroughFromPred and - // NewFreq. - NewFreq = 0; - FallThroughFromPred = 0; - } - } - - BlockFrequency Result = 0; - BlockFrequency Gains = BackEdgeFreq + NewFreq; - BlockFrequency Lost = FallThrough2Top + FallThrough2Exit + - FallThroughFromPred; - if (Gains > Lost) - Result = Gains - Lost; - return Result; + MachineBasicBlock *BestPred = nullptr; + BlockFrequency FallThroughFromPred = BlockFrequency(0); + for (MachineBasicBlock *Pred : NewTop->predecessors()) { + if (!LoopBlockSet.count(Pred)) + continue; + BlockChain *PredChain = BlockToChain[Pred]; + if (!PredChain || Pred == *std::prev(PredChain->end())) { + BlockFrequency EdgeFreq = + MBFI->getBlockFreq(Pred) * MBPI->getEdgeProbability(Pred, NewTop); + if (EdgeFreq > FallThroughFromPred) { + FallThroughFromPred = EdgeFreq; + BestPred = Pred; + } + } + } + + // If NewTop is not placed after Pred, another successor can be placed + // after Pred. + BlockFrequency NewFreq = BlockFrequency(0); + if (BestPred) { + for (MachineBasicBlock *Succ : BestPred->successors()) { + if ((Succ == NewTop) || (Succ == BestPred) || !LoopBlockSet.count(Succ)) + continue; + if (ComputedEdges.contains(Succ)) + continue; + BlockChain *SuccChain = BlockToChain[Succ]; + if ((SuccChain && (Succ != *SuccChain->begin())) || + (SuccChain == BlockToChain[BestPred])) + continue; + BlockFrequency EdgeFreq = MBFI->getBlockFreq(BestPred) * + MBPI->getEdgeProbability(BestPred, Succ); + if (EdgeFreq > NewFreq) + NewFreq = EdgeFreq; + } + BlockFrequency OrigEdgeFreq = MBFI->getBlockFreq(BestPred) * + MBPI->getEdgeProbability(BestPred, NewTop); + if (NewFreq > OrigEdgeFreq) { + // If NewTop is not the best successor of Pred, then Pred doesn't + // fallthrough to NewTop. So there is no FallThroughFromPred and + // NewFreq. + NewFreq = BlockFrequency(0); + FallThroughFromPred = BlockFrequency(0); + } + } + + BlockFrequency Result = BlockFrequency(0); + BlockFrequency Gains = BackEdgeFreq + NewFreq; + BlockFrequency Lost = + FallThrough2Top + FallThrough2Exit + FallThroughFromPred; + if (Gains > Lost) + Result = Gains - Lost; + return Result; } /// Helper function of findBestLoopTop. Find the best loop top block @@ -2087,7 +2087,7 @@ MachineBlockPlacement::findBestLoopTopHelper( LLVM_DEBUG(dbgs() << "Finding best loop top for: " << getBlockName(OldTop) << "\n"); - BlockFrequency BestGains = 0; + BlockFrequency BestGains = BlockFrequency(0); MachineBasicBlock *BestPred = nullptr; for (MachineBasicBlock *Pred : OldTop->predecessors()) { if (!LoopBlockSet.count(Pred)) @@ -2112,8 +2112,9 @@ MachineBlockPlacement::findBestLoopTopHelper( BlockFrequency Gains = FallThroughGains(Pred, OldTop, OtherBB, LoopBlockSet); - if ((Gains > 0) && (Gains > BestGains || - ((Gains == BestGains) && Pred->isLayoutSuccessor(OldTop)))) { + if ((Gains > BlockFrequency(0)) && + (Gains > BestGains || + ((Gains == BestGains) && Pred->isLayoutSuccessor(OldTop)))) { BestPred = Pred; BestGains = Gains; } @@ -2425,14 +2426,14 @@ void MachineBlockPlacement::rotateLoopWithProfile( if (ChainHeaderBB->isEntryBlock()) return; - BlockFrequency SmallestRotationCost = BlockFrequency::getMaxFrequency(); + BlockFrequency SmallestRotationCost = BlockFrequency::max(); // A utility lambda that scales up a block frequency by dividing it by a // branch probability which is the reciprocal of the scale. auto ScaleBlockFrequency = [](BlockFrequency Freq, unsigned Scale) -> BlockFrequency { if (Scale == 0) - return 0; + return BlockFrequency(0); // Use operator / between BlockFrequency and BranchProbability to implement // saturating multiplication. return Freq / BranchProbability(1, Scale); @@ -2492,7 +2493,7 @@ void MachineBlockPlacement::rotateLoopWithProfile( auto TailBB = *TailIter; // Calculate the cost by putting this BB to the top. - BlockFrequency Cost = 0; + BlockFrequency Cost = BlockFrequency(0); // If the current BB is the loop header, we need to take into account the // cost of the missed fall through edge from outside of the loop to the @@ -2523,8 +2524,7 @@ void MachineBlockPlacement::rotateLoopWithProfile( if (TailBB->isSuccessor(*Iter)) { auto TailBBFreq = MBFI->getBlockFreq(TailBB); if (TailBB->succ_size() == 1) - Cost += ScaleBlockFrequency(TailBBFreq.getFrequency(), - MisfetchCost + JumpInstCost); + Cost += ScaleBlockFrequency(TailBBFreq, MisfetchCost + JumpInstCost); else if (TailBB->succ_size() == 2) { auto TailToHeadProb = MBPI->getEdgeProbability(TailBB, *Iter); auto TailToHeadFreq = TailBBFreq * TailToHeadProb; @@ -3159,7 +3159,7 @@ static uint64_t countMBBInstruction(MachineBasicBlock *MBB) { // So we should scale the threshold accordingly. But the instruction size is not // available on all targets, so we use the number of instructions instead. BlockFrequency MachineBlockPlacement::scaleThreshold(MachineBasicBlock *BB) { - return DupThreshold.getFrequency() * countMBBInstruction(BB); + return BlockFrequency(DupThreshold.getFrequency() * countMBBInstruction(BB)); } // Returns true if BB is Pred's best successor. @@ -3313,7 +3313,7 @@ void MachineBlockPlacement::findDuplicateCandidates( } void MachineBlockPlacement::initDupThreshold() { - DupThreshold = 0; + DupThreshold = BlockFrequency(0); if (!F->getFunction().hasProfileData()) return; @@ -3321,12 +3321,13 @@ void MachineBlockPlacement::initDupThreshold() { uint64_t HotThreshold = PSI->getOrCompHotCountThreshold(); if (HotThreshold != UINT64_MAX) { UseProfileCount = true; - DupThreshold = HotThreshold * TailDupProfilePercentThreshold / 100; + DupThreshold = + BlockFrequency(HotThreshold * TailDupProfilePercentThreshold / 100); return; } // Profile count is not available, we can use block frequency instead. - BlockFrequency MaxFreq = 0; + BlockFrequency MaxFreq = BlockFrequency(0); for (MachineBasicBlock &MBB : *F) { BlockFrequency Freq = MBFI->getBlockFreq(&MBB); if (Freq > MaxFreq) @@ -3334,7 +3335,7 @@ void MachineBlockPlacement::initDupThreshold() { } BranchProbability ThresholdProb(TailDupPlacementPenalty, 100); - DupThreshold = MaxFreq * ThresholdProb; + DupThreshold = BlockFrequency(MaxFreq * ThresholdProb); UseProfileCount = false; } |