diff options
author | ShatianWang <38512325+ShatianWang@users.noreply.github.com> | 2023-11-29 22:42:36 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-11-29 22:42:36 -0500 |
commit | c43d0432ef35ee506bae642408f7140064514c8d (patch) | |
tree | 11efbf7ca7952742cc226b7bae34021a41b632d4 /bolt | |
parent | 3cf190974a37d18c7bd6f52b9588f4104e8e96f0 (diff) | |
download | llvm-c43d0432ef35ee506bae642408f7140064514c8d.zip llvm-c43d0432ef35ee506bae642408f7140064514c8d.tar.gz llvm-c43d0432ef35ee506bae642408f7140064514c8d.tar.bz2 |
[BOLT] Create .text.warm for 3-way splitting (#73863)
This commit explicitly adds a warm code section, .text.warm, when
-split-functions -split-strategy=cdsplit is used. This replaces the
previous approach of using .text.cold.0 as warm and .text.cold.1 as cold
in 3-way function splitting. NFC.
Diffstat (limited to 'bolt')
-rw-r--r-- | bolt/include/bolt/Core/BinaryContext.h | 7 | ||||
-rw-r--r-- | bolt/include/bolt/Core/BinaryFunction.h | 2 | ||||
-rw-r--r-- | bolt/include/bolt/Core/FunctionLayout.h | 1 | ||||
-rw-r--r-- | bolt/lib/Core/BinaryEmitter.cpp | 5 | ||||
-rw-r--r-- | bolt/lib/Passes/SplitFunctions.cpp | 12 | ||||
-rw-r--r-- | bolt/lib/Rewrite/RewriteInstance.cpp | 47 |
6 files changed, 57 insertions, 17 deletions
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h index 48a8b2a..c767228 100644 --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -614,6 +614,11 @@ public: /// Indicates if the function ordering of the binary is finalized. bool HasFinalizedFunctionOrder{false}; + /// Indicates if a separate .text.warm section is needed that contains + /// function fragments with + /// FunctionFragment::getFragmentNum() == FragmentNum::warm() + bool HasWarmSection{false}; + /// Is the binary always loaded at a fixed address. Shared objects and /// position-independent executables (PIEs) are examples of binaries that /// will have HasFixedLoadAddress set to false. @@ -930,6 +935,8 @@ public: const char *getMainCodeSectionName() const { return ".text"; } + const char *getWarmCodeSectionName() const { return ".text.warm"; } + const char *getColdCodeSectionName() const { return ".text.cold"; } const char *getHotTextMoverSectionName() const { return ".text.mover"; } diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h index 72c360c..182d5ff 100644 --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -1236,6 +1236,8 @@ public: return SmallString<32>(CodeSectionName); if (Fragment == FragmentNum::cold()) return SmallString<32>(ColdCodeSectionName); + if (BC.HasWarmSection && Fragment == FragmentNum::warm()) + return SmallString<32>(BC.getWarmCodeSectionName()); return formatv("{0}.{1}", ColdCodeSectionName, Fragment.get() - 1); } diff --git a/bolt/include/bolt/Core/FunctionLayout.h b/bolt/include/bolt/Core/FunctionLayout.h index 904da3a..2e4c184 100644 --- a/bolt/include/bolt/Core/FunctionLayout.h +++ b/bolt/include/bolt/Core/FunctionLayout.h @@ -63,6 +63,7 @@ public: static constexpr FragmentNum main() { return FragmentNum(0); } static constexpr FragmentNum cold() { return FragmentNum(1); } + static constexpr FragmentNum warm() { return FragmentNum(2); } }; /// A freestanding subset of contiguous blocks of a function. diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp index fb1bf53..3bff312 100644 --- a/bolt/lib/Core/BinaryEmitter.cpp +++ b/bolt/lib/Core/BinaryEmitter.cpp @@ -287,7 +287,10 @@ void BinaryEmitter::emitFunctions() { // Mark the end of hot text. if (opts::HotText) { - Streamer.switchSection(BC.getTextSection()); + if (BC.HasWarmSection) + Streamer.switchSection(BC.getCodeSection(BC.getWarmCodeSectionName())); + else + Streamer.switchSection(BC.getTextSection()); Streamer.emitLabel(BC.getHotTextEndSymbol()); } } diff --git a/bolt/lib/Passes/SplitFunctions.cpp b/bolt/lib/Passes/SplitFunctions.cpp index af2a5aa..79da4cc 100644 --- a/bolt/lib/Passes/SplitFunctions.cpp +++ b/bolt/lib/Passes/SplitFunctions.cpp @@ -160,16 +160,15 @@ struct SplitCacheDirected final : public SplitStrategy { // Assign fragments based on the computed best split index. // All basic blocks with index up to the best split index become hot. // All remaining blocks are warm / cold depending on if count is - // greater than 0 or not. - FragmentNum Main(0); - FragmentNum Cold(1); - FragmentNum Warm(2); + // greater than zero or not. for (size_t Index = 0; Index < BlockOrder.size(); Index++) { BinaryBasicBlock *BB = BlockOrder[Index]; if (Index <= BestSplitIndex) - BB->setFragmentNum(Main); + BB->setFragmentNum(FragmentNum::main()); else - BB->setFragmentNum(BB->getKnownExecutionCount() > 0 ? Warm : Cold); + BB->setFragmentNum(BB->getKnownExecutionCount() > 0 + ? FragmentNum::warm() + : FragmentNum::cold()); } } @@ -313,6 +312,7 @@ void SplitFunctions::runOnFunctions(BinaryContext &BC) { else Strategy = std::make_unique<SplitProfile2>(); opts::AggressiveSplitting = true; + BC.HasWarmSection = true; break; case SplitFunctionsStrategy::Profile2: Strategy = std::make_unique<SplitProfile2>(); diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index 81c9cbf..8cda0b7 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -3478,11 +3478,21 @@ std::vector<BinarySection *> RewriteInstance::getCodeSections() { if (B->getName() == BC->getHotTextMoverSectionName()) return false; - // Depending on the option, put main text at the beginning or at the end. - if (opts::HotFunctionsAtEnd) - return B->getName() == BC->getMainCodeSectionName(); - else - return A->getName() == BC->getMainCodeSectionName(); + // Depending on opts::HotFunctionsAtEnd, place main and warm sections in + // order. + if (opts::HotFunctionsAtEnd) { + if (B->getName() == BC->getMainCodeSectionName()) + return true; + if (A->getName() == BC->getMainCodeSectionName()) + return false; + return (B->getName() == BC->getWarmCodeSectionName()); + } else { + if (A->getName() == BC->getMainCodeSectionName()) + return true; + if (B->getName() == BC->getMainCodeSectionName()) + return false; + return (A->getName() == BC->getWarmCodeSectionName()); + } }; // Determine the order of sections. @@ -3524,6 +3534,9 @@ void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) { // Allocate sections starting at a given Address. auto allocateAt = [&](uint64_t Address) { + const char *LastNonColdSectionName = BC->HasWarmSection + ? BC->getWarmCodeSectionName() + : BC->getMainCodeSectionName(); for (BinarySection *Section : CodeSections) { Address = alignTo(Address, Section->getAlignment()); Section->setOutputAddress(Address); @@ -3532,13 +3545,13 @@ void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) { // Hugify: Additional huge page from right side due to // weird ASLR mapping addresses (4KB aligned) if (opts::Hugify && !BC->HasFixedLoadAddress && - Section->getName() == BC->getMainCodeSectionName()) + Section->getName() == LastNonColdSectionName) Address = alignTo(Address, Section->getAlignment()); } // Make sure we allocate enough space for huge pages. ErrorOr<BinarySection &> TextSection = - BC->getUniqueSectionByName(BC->getMainCodeSectionName()); + BC->getUniqueSectionByName(LastNonColdSectionName); if (opts::HotText && TextSection && TextSection->hasValidSectionID()) { uint64_t HotTextEnd = TextSection->getOutputAddress() + TextSection->getOutputSize(); @@ -4396,6 +4409,21 @@ void RewriteInstance::updateELFSymbolTable( return NewIndex; }; + // Get the extra symbol name of a split fragment; used in addExtraSymbols. + auto getSplitSymbolName = [&](const FunctionFragment &FF, + const ELFSymTy &FunctionSymbol) { + SmallString<256> SymbolName; + if (BC->HasWarmSection) + SymbolName = + formatv("{0}.{1}", cantFail(FunctionSymbol.getName(StringSection)), + FF.getFragmentNum() == FragmentNum::warm() ? "warm" : "cold"); + else + SymbolName = formatv("{0}.cold.{1}", + cantFail(FunctionSymbol.getName(StringSection)), + FF.getFragmentNum().get() - 1); + return SymbolName; + }; + // Add extra symbols for the function. // // Note that addExtraSymbols() could be called multiple times for the same @@ -4423,9 +4451,8 @@ void RewriteInstance::updateELFSymbolTable( Function.getLayout().getSplitFragments()) { if (FF.getAddress()) { ELFSymTy NewColdSym = FunctionSymbol; - const SmallString<256> SymbolName = formatv( - "{0}.cold.{1}", cantFail(FunctionSymbol.getName(StringSection)), - FF.getFragmentNum().get() - 1); + const SmallString<256> SymbolName = + getSplitSymbolName(FF, FunctionSymbol); NewColdSym.st_name = AddToStrTab(SymbolName); NewColdSym.st_shndx = Function.getCodeSection(FF.getFragmentNum())->getIndex(); |