aboutsummaryrefslogtreecommitdiff
path: root/bolt/lib
diff options
context:
space:
mode:
Diffstat (limited to 'bolt/lib')
-rw-r--r--bolt/lib/Passes/LongJmp.cpp4
-rw-r--r--bolt/lib/Passes/SplitFunctions.cpp35
-rw-r--r--bolt/lib/Rewrite/CMakeLists.txt1
-rw-r--r--bolt/lib/Rewrite/GNUPropertyRewriter.cpp147
-rw-r--r--bolt/lib/Rewrite/RewriteInstance.cpp9
-rw-r--r--bolt/lib/Utils/CommandLineOpts.cpp23
6 files changed, 190 insertions, 29 deletions
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 4dade16..03c1ea9 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -895,6 +895,10 @@ void LongJmpPass::relaxLocalBranches(BinaryFunction &BF) {
Error LongJmpPass::runOnFunctions(BinaryContext &BC) {
+ assert((opts::CompactCodeModel ||
+ opts::SplitStrategy != opts::SplitFunctionsStrategy::CDSplit) &&
+ "LongJmp cannot work with functions split in more than two fragments");
+
if (opts::CompactCodeModel) {
BC.outs()
<< "BOLT-INFO: relaxing branches for compact code model (<128MB)\n";
diff --git a/bolt/lib/Passes/SplitFunctions.cpp b/bolt/lib/Passes/SplitFunctions.cpp
index b21401e..eab669b 100644
--- a/bolt/lib/Passes/SplitFunctions.cpp
+++ b/bolt/lib/Passes/SplitFunctions.cpp
@@ -86,29 +86,6 @@ static cl::opt<unsigned> SplitThreshold(
"increase after splitting."),
cl::init(0), cl::Hidden, cl::cat(BoltOptCategory));
-static cl::opt<SplitFunctionsStrategy> SplitStrategy(
- "split-strategy", cl::init(SplitFunctionsStrategy::Profile2),
- cl::values(clEnumValN(SplitFunctionsStrategy::Profile2, "profile2",
- "split each function into a hot and cold fragment "
- "using profiling information")),
- cl::values(clEnumValN(SplitFunctionsStrategy::CDSplit, "cdsplit",
- "split each function into a hot, warm, and cold "
- "fragment using profiling information")),
- cl::values(clEnumValN(
- SplitFunctionsStrategy::Random2, "random2",
- "split each function into a hot and cold fragment at a randomly chosen "
- "split point (ignoring any available profiling information)")),
- cl::values(clEnumValN(
- SplitFunctionsStrategy::RandomN, "randomN",
- "split each function into N fragments at a randomly chosen split "
- "points (ignoring any available profiling information)")),
- cl::values(clEnumValN(
- SplitFunctionsStrategy::All, "all",
- "split all basic blocks of each function into fragments such that each "
- "fragment contains exactly a single basic block")),
- cl::desc("strategy used to partition blocks into fragments"),
- cl::cat(BoltOptCategory));
-
static cl::opt<double> CallScale(
"call-scale",
cl::desc("Call score scale coefficient (when --split-strategy=cdsplit)"),
@@ -724,14 +701,14 @@ Error SplitFunctions::runOnFunctions(BinaryContext &BC) {
// If split strategy is not CDSplit, then a second run of the pass is not
// needed after function reordering.
if (BC.HasFinalizedFunctionOrder &&
- opts::SplitStrategy != SplitFunctionsStrategy::CDSplit)
+ opts::SplitStrategy != opts::SplitFunctionsStrategy::CDSplit)
return Error::success();
std::unique_ptr<SplitStrategy> Strategy;
bool ForceSequential = false;
switch (opts::SplitStrategy) {
- case SplitFunctionsStrategy::CDSplit:
+ case opts::SplitFunctionsStrategy::CDSplit:
// CDSplit runs two splitting passes: hot-cold splitting (SplitPrfoile2)
// before function reordering and hot-warm-cold splitting
// (SplitCacheDirected) after function reordering.
@@ -742,21 +719,21 @@ Error SplitFunctions::runOnFunctions(BinaryContext &BC) {
opts::AggressiveSplitting = true;
BC.HasWarmSection = true;
break;
- case SplitFunctionsStrategy::Profile2:
+ case opts::SplitFunctionsStrategy::Profile2:
Strategy = std::make_unique<SplitProfile2>();
break;
- case SplitFunctionsStrategy::Random2:
+ case opts::SplitFunctionsStrategy::Random2:
Strategy = std::make_unique<SplitRandom2>();
// If we split functions randomly, we need to ensure that across runs with
// the same input, we generate random numbers for each function in the same
// order.
ForceSequential = true;
break;
- case SplitFunctionsStrategy::RandomN:
+ case opts::SplitFunctionsStrategy::RandomN:
Strategy = std::make_unique<SplitRandomN>();
ForceSequential = true;
break;
- case SplitFunctionsStrategy::All:
+ case opts::SplitFunctionsStrategy::All:
Strategy = std::make_unique<SplitAll>();
break;
}
diff --git a/bolt/lib/Rewrite/CMakeLists.txt b/bolt/lib/Rewrite/CMakeLists.txt
index 7750360..5b15edc 100644
--- a/bolt/lib/Rewrite/CMakeLists.txt
+++ b/bolt/lib/Rewrite/CMakeLists.txt
@@ -25,6 +25,7 @@ add_llvm_library(LLVMBOLTRewrite
PseudoProbeRewriter.cpp
RewriteInstance.cpp
SDTRewriter.cpp
+ GNUPropertyRewriter.cpp
NO_EXPORT
DISABLE_LLVM_LINK_LLVM_DYLIB
diff --git a/bolt/lib/Rewrite/GNUPropertyRewriter.cpp b/bolt/lib/Rewrite/GNUPropertyRewriter.cpp
new file mode 100644
index 0000000..f61c08e
--- /dev/null
+++ b/bolt/lib/Rewrite/GNUPropertyRewriter.cpp
@@ -0,0 +1,147 @@
+//===- bolt/Rewrite/GNUPropertyRewriter.cpp -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Read the .note.gnu.property section.
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Rewrite/MetadataRewriter.h"
+#include "bolt/Rewrite/MetadataRewriters.h"
+#include "llvm/Support/Errc.h"
+
+using namespace llvm;
+using namespace bolt;
+
+namespace {
+
+class GNUPropertyRewriter final : public MetadataRewriter {
+
+ Expected<uint32_t> decodeGNUPropertyNote(StringRef Desc);
+
+public:
+ GNUPropertyRewriter(StringRef Name, BinaryContext &BC)
+ : MetadataRewriter(Name, BC) {}
+
+ Error sectionInitializer() override;
+};
+
+Error GNUPropertyRewriter::sectionInitializer() {
+
+ ErrorOr<BinarySection &> Sec =
+ BC.getUniqueSectionByName(".note.gnu.property");
+ if (!Sec)
+ return Error::success();
+
+ // Accumulate feature bits
+ uint32_t FeaturesAcc = 0;
+
+ StringRef Buf = Sec->getContents();
+ DataExtractor DE(Buf, BC.AsmInfo->isLittleEndian(),
+ BC.AsmInfo->getCodePointerSize());
+ DataExtractor::Cursor Cursor(0);
+ while (Cursor && !DE.eof(Cursor)) {
+ const uint32_t NameSz = DE.getU32(Cursor);
+ const uint32_t DescSz = DE.getU32(Cursor);
+ const uint32_t Type = DE.getU32(Cursor);
+
+ StringRef Name =
+ NameSz ? Buf.slice(Cursor.tell(), Cursor.tell() + NameSz) : "<empty>";
+ Cursor.seek(alignTo(Cursor.tell() + NameSz, 4));
+
+ const uint64_t DescOffset = Cursor.tell();
+ StringRef Desc =
+ DescSz ? Buf.slice(DescOffset, DescOffset + DescSz) : "<empty>";
+ Cursor.seek(alignTo(DescOffset + DescSz, 4));
+ if (!Cursor)
+ return createStringError(
+ errc::executable_format_error,
+ "out of bounds while reading .note.gnu.property section: %s",
+ toString(Cursor.takeError()).c_str());
+
+ if (Type == ELF::NT_GNU_PROPERTY_TYPE_0 && Name.starts_with("GNU") &&
+ DescSz) {
+ auto Features = decodeGNUPropertyNote(Desc);
+ if (!Features)
+ return Features.takeError();
+ FeaturesAcc |= *Features;
+ }
+ }
+
+ if (BC.isAArch64()) {
+ BC.setUsesBTI(FeaturesAcc & llvm::ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI);
+ if (BC.usesBTI())
+ BC.outs() << "BOLT-WARNING: binary is using BTI. Optimized binary may be "
+ "corrupted\n";
+ }
+
+ return Error::success();
+}
+
+/// \p Desc contains an array of property descriptors. Each member has the
+/// following structure:
+/// typedef struct {
+/// Elf_Word pr_type;
+/// Elf_Word pr_datasz;
+/// unsigned char pr_data[PR_DATASZ];
+/// unsigned char pr_padding[PR_PADDING];
+/// } Elf_Prop;
+///
+/// As there is no guarantee that the features are encoded in which element of
+/// the array, we have to read all, and OR together the result.
+Expected<uint32_t> GNUPropertyRewriter::decodeGNUPropertyNote(StringRef Desc) {
+ DataExtractor DE(Desc, BC.AsmInfo->isLittleEndian(),
+ BC.AsmInfo->getCodePointerSize());
+ DataExtractor::Cursor Cursor(0);
+ const uint32_t Align = DE.getAddressSize();
+
+ std::optional<uint32_t> Features = 0;
+ while (Cursor && !DE.eof(Cursor)) {
+ const uint32_t PrType = DE.getU32(Cursor);
+ const uint32_t PrDataSz = DE.getU32(Cursor);
+
+ const uint64_t PrDataStart = Cursor.tell();
+ const uint64_t PrDataEnd = PrDataStart + PrDataSz;
+ Cursor.seek(PrDataEnd);
+ if (!Cursor)
+ return createStringError(
+ errc::executable_format_error,
+ "out of bounds while reading .note.gnu.property section: %s",
+ toString(Cursor.takeError()).c_str());
+
+ if (PrType == llvm::ELF::GNU_PROPERTY_AARCH64_FEATURE_1_AND) {
+ if (PrDataSz != 4) {
+ return createStringError(
+ errc::executable_format_error,
+ "Property descriptor size has to be 4 bytes on AArch64\n");
+ }
+ DataExtractor::Cursor Tmp(PrDataStart);
+ // PrDataSz = 4 -> PrData is uint32_t
+ const uint32_t FeaturesItem = DE.getU32(Tmp);
+ if (!Tmp)
+ return createStringError(
+ errc::executable_format_error,
+ "failed to read property from .note.gnu.property section: %s",
+ toString(Tmp.takeError()).c_str());
+ Features = Features ? (*Features | FeaturesItem) : FeaturesItem;
+ }
+
+ Cursor.seek(alignTo(PrDataEnd, Align));
+ if (!Cursor)
+ return createStringError(errc::executable_format_error,
+ "out of bounds while reading property array in "
+ ".note.gnu.property section: %s",
+ toString(Cursor.takeError()).c_str());
+ }
+ return Features.value_or(0u);
+}
+} // namespace
+
+std::unique_ptr<MetadataRewriter>
+llvm::bolt::createGNUPropertyRewriter(BinaryContext &BC) {
+ return std::make_unique<GNUPropertyRewriter>("gnu-property-rewriter", BC);
+}
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index c13a9f0..bfd03e0 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -2115,6 +2115,13 @@ void RewriteInstance::adjustCommandLineOptions() {
opts::SplitEH = false;
}
+ if (BC->isAArch64() && !opts::CompactCodeModel &&
+ opts::SplitStrategy == opts::SplitFunctionsStrategy::CDSplit) {
+ BC->errs() << "BOLT-ERROR: CDSplit is not supported with LongJmp. Try with "
+ "'--compact-code-model'\n";
+ exit(1);
+ }
+
if (opts::StrictMode && !BC->HasRelocations) {
BC->errs()
<< "BOLT-WARNING: disabling strict mode (-strict) in non-relocation "
@@ -3331,6 +3338,8 @@ void RewriteInstance::initializeMetadataManager() {
MetadataManager.registerRewriter(createPseudoProbeRewriter(*BC));
MetadataManager.registerRewriter(createSDTRewriter(*BC));
+
+ MetadataManager.registerRewriter(createGNUPropertyRewriter(*BC));
}
void RewriteInstance::processSectionMetadata() {
diff --git a/bolt/lib/Utils/CommandLineOpts.cpp b/bolt/lib/Utils/CommandLineOpts.cpp
index 5635da4..095612a 100644
--- a/bolt/lib/Utils/CommandLineOpts.cpp
+++ b/bolt/lib/Utils/CommandLineOpts.cpp
@@ -104,6 +104,29 @@ ExecutionCountThreshold("execution-count-threshold",
cl::Hidden,
cl::cat(BoltOptCategory));
+cl::opt<SplitFunctionsStrategy> SplitStrategy(
+ "split-strategy", cl::init(SplitFunctionsStrategy::Profile2),
+ cl::values(clEnumValN(SplitFunctionsStrategy::Profile2, "profile2",
+ "split each function into a hot and cold fragment "
+ "using profiling information")),
+ cl::values(clEnumValN(SplitFunctionsStrategy::CDSplit, "cdsplit",
+ "split each function into a hot, warm, and cold "
+ "fragment using profiling information")),
+ cl::values(clEnumValN(
+ SplitFunctionsStrategy::Random2, "random2",
+ "split each function into a hot and cold fragment at a randomly chosen "
+ "split point (ignoring any available profiling information)")),
+ cl::values(clEnumValN(
+ SplitFunctionsStrategy::RandomN, "randomN",
+ "split each function into N fragments at a randomly chosen split "
+ "points (ignoring any available profiling information)")),
+ cl::values(clEnumValN(
+ SplitFunctionsStrategy::All, "all",
+ "split all basic blocks of each function into fragments such that each "
+ "fragment contains exactly a single basic block")),
+ cl::desc("strategy used to partition blocks into fragments"),
+ cl::cat(BoltOptCategory));
+
bool HeatmapBlockSpecParser::parse(cl::Option &O, StringRef ArgName,
StringRef Arg, HeatmapBlockSizes &Val) {
// Parses a human-readable suffix into a shift amount or nullopt on error.
generated by cgit v1.1 at 2025-10-03 17:08:36 +0000