aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--bolt/include/bolt/Core/BinaryContext.h20
-rw-r--r--bolt/include/bolt/Core/JumpTable.h28
-rw-r--r--bolt/lib/Core/BinaryContext.cpp136
-rw-r--r--bolt/lib/Core/BinaryEmitter.cpp2
-rw-r--r--bolt/lib/Core/BinaryFunction.cpp17
-rw-r--r--bolt/lib/Core/JumpTable.cpp6
-rw-r--r--bolt/lib/Passes/IndirectCallPromotion.cpp4
-rw-r--r--bolt/lib/Passes/JTFootprintReduction.cpp2
8 files changed, 119 insertions, 96 deletions
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 485979f..69b8f02 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -435,7 +435,18 @@ public:
/// Return size of an entry for the given jump table \p Type.
uint64_t getJumpTableEntrySize(JumpTable::JumpTableType Type) const {
- return Type == JumpTable::JTT_PIC ? 4 : AsmInfo->getCodePointerSize();
+ switch (Type) {
+ case JumpTable::JTT_X86_64_PIC4:
+ return 4;
+ case JumpTable::JTT_X86_64_ABS:
+ return AsmInfo->getCodePointerSize();
+ case JumpTable::JTT_AARCH64_REL1:
+ return 1;
+ case JumpTable::JTT_AARCH64_REL2:
+ return 2;
+ case JumpTable::JTT_AARCH64_REL4:
+ return 4;
+ }
}
/// Return JumpTable containing a given \p Address.
@@ -573,14 +584,13 @@ public:
/// If \p NextJTAddress is different from zero, it is used as an upper
/// bound for jump table memory layout.
///
- /// Optionally, populate \p Address from jump table entries. The entries
- /// could be partially populated if the jump table detection fails.
+ /// If \p JT is set, populate it with jump table entries. The entries could be
+ /// partially populated if the jump table detection fails.
bool analyzeJumpTable(const uint64_t Address,
const JumpTable::JumpTableType Type,
const BinaryFunction &BF,
const uint64_t NextJTAddress = 0,
- JumpTable::AddressesType *EntriesAsAddress = nullptr,
- bool *HasEntryInFragment = nullptr) const;
+ JumpTable *JT = nullptr) const;
/// After jump table locations are established, this function will populate
/// their EntriesAsAddress based on memory contents.
diff --git a/bolt/include/bolt/Core/JumpTable.h b/bolt/include/bolt/Core/JumpTable.h
index 52b9cce..7dc0c65 100644
--- a/bolt/include/bolt/Core/JumpTable.h
+++ b/bolt/include/bolt/Core/JumpTable.h
@@ -47,10 +47,34 @@ class JumpTable : public BinaryData {
public:
enum JumpTableType : char {
- JTT_NORMAL,
- JTT_PIC,
+ JTT_X86_64_FIRST = 0,
+ JTT_X86_64_ABS = JTT_X86_64_FIRST,
+ JTT_X86_64_PIC4,
+ JTT_X86_64_LAST = JTT_X86_64_PIC4,
+ JTT_AARCH64_FIRST,
+ JTT_AARCH64_REL1 = JTT_AARCH64_FIRST,
+ JTT_AARCH64_REL2,
+ JTT_AARCH64_REL4,
+ JTT_AARCH64_LAST = JTT_AARCH64_REL4
};
+ static StringRef getTypeStr(JumpTableType Type) {
+ switch (Type) {
+ case JTT_X86_64_ABS:
+ return "X86_64_ABS";
+ case JTT_X86_64_PIC4:
+ return "X86_64_PIC4";
+ case JTT_AARCH64_REL1:
+ return "AARCH64_REL1";
+ case JTT_AARCH64_REL2:
+ return "AARCH64_REL2";
+ case JTT_AARCH64_REL4:
+ return "AARCH64_REL4";
+ }
+ }
+
+ const StringRef getTypeStr() { return getTypeStr(Type); }
+
/// Branch statistics for jump table entries.
struct JumpInfo {
uint64_t Mispreds{0};
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 8fa1e36..cf214ac 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -496,7 +496,7 @@ BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF,
const MemoryContentsType MemType = analyzeMemoryAt(Address, BF);
if (MemType == MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE && IsPCRel) {
const MCSymbol *Symbol =
- getOrCreateJumpTable(BF, Address, JumpTable::JTT_PIC);
+ getOrCreateJumpTable(BF, Address, JumpTable::JTT_X86_64_PIC4);
return std::make_pair(Symbol, 0);
}
@@ -540,10 +540,10 @@ MemoryContentsType BinaryContext::analyzeMemoryAt(uint64_t Address,
// Start with checking for PIC jump table. We expect non-PIC jump tables
// to have high 32 bits set to 0.
- if (analyzeJumpTable(Address, JumpTable::JTT_PIC, BF))
+ if (analyzeJumpTable(Address, JumpTable::JTT_X86_64_PIC4, BF))
return MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE;
- if (analyzeJumpTable(Address, JumpTable::JTT_NORMAL, BF))
+ if (analyzeJumpTable(Address, JumpTable::JTT_X86_64_ABS, BF))
return MemoryContentsType::POSSIBLE_JUMP_TABLE;
return MemoryContentsType::UNKNOWN;
@@ -553,8 +553,7 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
const JumpTable::JumpTableType Type,
const BinaryFunction &BF,
const uint64_t NextJTAddress,
- JumpTable::AddressesType *EntriesAsAddress,
- bool *HasEntryInFragment) const {
+ JumpTable *JT) const {
// Target address of __builtin_unreachable.
const uint64_t UnreachableAddress = BF.getAddress() + BF.getSize();
@@ -571,11 +570,11 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
size_t TrimmedSize = 0;
auto addEntryAddress = [&](uint64_t EntryAddress, bool Unreachable = false) {
- if (!EntriesAsAddress)
+ if (!JT)
return;
- EntriesAsAddress->emplace_back(EntryAddress);
+ JT->EntriesAsAddress.emplace_back(EntryAddress);
if (!Unreachable)
- TrimmedSize = EntriesAsAddress->size();
+ TrimmedSize = JT->EntriesAsAddress.size();
};
ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);
@@ -594,12 +593,9 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
if (NextJTAddress)
UpperBound = std::min(NextJTAddress, UpperBound);
- LLVM_DEBUG({
- using JTT = JumpTable::JumpTableType;
- dbgs() << formatv("BOLT-DEBUG: analyzeJumpTable @{0:x} in {1}, JTT={2}\n",
- Address, BF.getPrintName(),
- Type == JTT::JTT_PIC ? "PIC" : "Normal");
- });
+ LLVM_DEBUG(
+ dbgs() << formatv("BOLT-DEBUG: analyzeJumpTable @{0:x} in {1}, JTT={2}\n",
+ Address, BF, JumpTable::getTypeStr(Type)));
const uint64_t EntrySize = getJumpTableEntrySize(Type);
for (uint64_t EntryAddress = Address; EntryAddress <= UpperBound - EntrySize;
EntryAddress += EntrySize) {
@@ -607,13 +603,13 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
<< " -> ");
// Check if there's a proper relocation against the jump table entry.
if (HasRelocations) {
- if (Type == JumpTable::JTT_PIC &&
+ if (Type == JumpTable::JTT_X86_64_PIC4 &&
!DataPCRelocations.count(EntryAddress)) {
LLVM_DEBUG(
dbgs() << "FAIL: JTT_PIC table, no relocation for this address\n");
break;
}
- if (Type == JumpTable::JTT_NORMAL && !getRelocationAt(EntryAddress)) {
+ if (Type == JumpTable::JTT_X86_64_ABS && !getRelocationAt(EntryAddress)) {
LLVM_DEBUG(
dbgs()
<< "FAIL: JTT_NORMAL table, no relocation for this address\n");
@@ -621,10 +617,15 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
}
}
- const uint64_t Value =
- (Type == JumpTable::JTT_PIC)
- ? Address + *getSignedValueAtAddress(EntryAddress, EntrySize)
- : *getPointerAtAddress(EntryAddress);
+ uint64_t Value = 0;
+ switch (Type) {
+ case JumpTable::JTT_X86_64_PIC4:
+ Value = Address + *getSignedValueAtAddress(EntryAddress, EntrySize);
+ break;
+ case JumpTable::JTT_X86_64_ABS:
+ Value = *getPointerAtAddress(EntryAddress);
+ break;
+ }
// __builtin_unreachable() case.
if (Value == UnreachableAddress) {
@@ -645,24 +646,19 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
// Function or one of its fragments.
const BinaryFunction *TargetBF = getBinaryFunctionContainingAddress(Value);
- const bool DoesBelongToFunction =
- BF.containsAddress(Value) ||
- (TargetBF && areRelatedFragments(TargetBF, &BF));
- if (!DoesBelongToFunction) {
+ if (!TargetBF || !areRelatedFragments(TargetBF, &BF)) {
LLVM_DEBUG({
- if (!BF.containsAddress(Value)) {
- dbgs() << "FAIL: function doesn't contain this address\n";
- if (TargetBF) {
- dbgs() << " ! function containing this address: "
- << TargetBF->getPrintName() << '\n';
- if (TargetBF->isFragment()) {
- dbgs() << " ! is a fragment";
- for (BinaryFunction *Parent : TargetBF->ParentFragments)
- dbgs() << ", parent: " << Parent->getPrintName();
- dbgs() << '\n';
- }
- }
- }
+ dbgs() << "FAIL: function doesn't contain this address\n";
+ if (!TargetBF)
+ break;
+ dbgs() << " ! function containing this address: " << *TargetBF << '\n';
+ if (!TargetBF->isFragment())
+ break;
+ dbgs() << " ! is a fragment with parents: ";
+ ListSeparator LS;
+ for (BinaryFunction *Parent : TargetBF->ParentFragments)
+ dbgs() << LS << *Parent;
+ dbgs() << '\n';
});
break;
}
@@ -677,17 +673,17 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
++NumRealEntries;
LLVM_DEBUG(dbgs() << formatv("OK: {0:x} real entry\n", Value));
- if (TargetBF != &BF && HasEntryInFragment)
- *HasEntryInFragment = true;
+ if (TargetBF != &BF && JT)
+ JT->IsSplit = true;
addEntryAddress(Value);
}
// Trim direct/normal jump table to exclude trailing unreachable entries that
// can collide with a function address.
- if (Type == JumpTable::JTT_NORMAL && EntriesAsAddress &&
- TrimmedSize != EntriesAsAddress->size() &&
+ if (Type == JumpTable::JTT_X86_64_ABS && JT &&
+ TrimmedSize != JT->EntriesAsAddress.size() &&
getBinaryFunctionAtAddress(UnreachableAddress))
- EntriesAsAddress->resize(TrimmedSize);
+ JT->EntriesAsAddress.resize(TrimmedSize);
// It's a jump table if the number of real entries is more than 1, or there's
// one real entry and one or more special targets. If there are only multiple
@@ -702,10 +698,8 @@ void BinaryContext::populateJumpTables() {
++JTI) {
JumpTable *JT = JTI->second;
- bool NonSimpleParent = false;
- for (BinaryFunction *BF : JT->Parents)
- NonSimpleParent |= !BF->isSimple();
- if (NonSimpleParent)
+ auto isSimple = std::bind(&BinaryFunction::isSimple, std::placeholders::_1);
+ if (!llvm::all_of(JT->Parents, isSimple))
continue;
uint64_t NextJTAddress = 0;
@@ -713,9 +707,8 @@ void BinaryContext::populateJumpTables() {
if (NextJTI != JTE)
NextJTAddress = NextJTI->second->getAddress();
- const bool Success =
- analyzeJumpTable(JT->getAddress(), JT->Type, *(JT->Parents[0]),
- NextJTAddress, &JT->EntriesAsAddress, &JT->IsSplit);
+ const bool Success = analyzeJumpTable(
+ JT->getAddress(), JT->Type, *JT->Parents.front(), NextJTAddress, JT);
if (!Success) {
LLVM_DEBUG({
dbgs() << "failed to analyze ";
@@ -743,7 +736,7 @@ void BinaryContext::populateJumpTables() {
// In strict mode, erase PC-relative relocation record. Later we check that
// all such records are erased and thus have been accounted for.
- if (opts::StrictMode && JT->Type == JumpTable::JTT_PIC) {
+ if (opts::StrictMode && JT->Type == JumpTable::JTT_X86_64_PIC4) {
for (uint64_t Address = JT->getAddress();
Address < JT->getAddress() + JT->getSize();
Address += JT->EntrySize) {
@@ -839,33 +832,26 @@ BinaryContext::getOrCreateJumpTable(BinaryFunction &Function, uint64_t Address,
assert(JT->Type == Type && "jump table types have to match");
assert(Address == JT->getAddress() && "unexpected non-empty jump table");
- // Prevent associating a jump table to a specific fragment twice.
- if (!llvm::is_contained(JT->Parents, &Function)) {
- assert(llvm::all_of(JT->Parents,
- [&](const BinaryFunction *BF) {
- return areRelatedFragments(&Function, BF);
- }) &&
- "cannot re-use jump table of a different function");
- // Duplicate the entry for the parent function for easy access
- JT->Parents.push_back(&Function);
- if (opts::Verbosity > 2) {
- this->outs() << "BOLT-INFO: Multiple fragments access same jump table: "
- << JT->Parents[0]->getPrintName() << "; "
- << Function.getPrintName() << "\n";
- JT->print(this->outs());
- }
- Function.JumpTables.emplace(Address, JT);
- for (BinaryFunction *Parent : JT->Parents)
- Parent->setHasIndirectTargetToSplitFragment(true);
- }
+ if (llvm::is_contained(JT->Parents, &Function))
+ return JT->getFirstLabel();
- bool IsJumpTableParent = false;
- (void)IsJumpTableParent;
- for (BinaryFunction *Frag : JT->Parents)
- if (Frag == &Function)
- IsJumpTableParent = true;
- assert(IsJumpTableParent &&
+ // Prevent associating a jump table to a specific fragment twice.
+ auto isSibling = std::bind(&BinaryContext::areRelatedFragments, this,
+ &Function, std::placeholders::_1);
+ assert(llvm::all_of(JT->Parents, isSibling) &&
"cannot re-use jump table of a different function");
+ if (opts::Verbosity > 2) {
+ this->outs() << "BOLT-INFO: Multiple fragments access same jump table: "
+ << JT->Parents[0]->getPrintName() << "; "
+ << Function.getPrintName() << "\n";
+ JT->print(this->outs());
+ }
+ if (JT->Parents.size() == 1)
+ JT->Parents.front()->setHasIndirectTargetToSplitFragment(true);
+ Function.setHasIndirectTargetToSplitFragment(true);
+ // Duplicate the entry for the parent function for easy access
+ JT->Parents.push_back(&Function);
+ Function.JumpTables.emplace(Address, JT);
return JT->getFirstLabel();
}
diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp
index 1aad252..7727204 100644
--- a/bolt/lib/Core/BinaryEmitter.cpp
+++ b/bolt/lib/Core/BinaryEmitter.cpp
@@ -850,7 +850,7 @@ void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection,
}
LastLabel = LI->second;
}
- if (JT.Type == JumpTable::JTT_NORMAL) {
+ if (JT.Type == JumpTable::JTT_X86_64_ABS) {
Streamer.emitSymbolValue(Entry, JT.OutputEntrySize);
} else { // JTT_PIC
const MCSymbolRefExpr *JTExpr =
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index b46ba1a..effd3c4 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -912,7 +912,7 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
"Invalid memory instruction");
const MCExpr *FixedEntryDispExpr = FixedEntryDispOperand->getExpr();
const uint64_t EntryAddress = getExprValue(FixedEntryDispExpr);
- uint64_t EntrySize = BC.getJumpTableEntrySize(JumpTable::JTT_PIC);
+ uint64_t EntrySize = BC.getJumpTableEntrySize(JumpTable::JTT_X86_64_PIC4);
ErrorOr<int64_t> Value =
BC.getSignedValueAtAddress(EntryAddress, EntrySize);
if (!Value)
@@ -982,12 +982,14 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
MemoryContentsType MemType;
if (JumpTable *JT = BC.getJumpTableContainingAddress(ArrayStart)) {
switch (JT->Type) {
- case JumpTable::JTT_NORMAL:
+ case JumpTable::JTT_X86_64_ABS:
MemType = MemoryContentsType::POSSIBLE_JUMP_TABLE;
break;
- case JumpTable::JTT_PIC:
+ case JumpTable::JTT_X86_64_PIC4:
MemType = MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE;
break;
+ default:
+ llvm_unreachable("Unhandled jump table type");
}
} else {
MemType = BC.analyzeMemoryAt(ArrayStart, *this);
@@ -998,7 +1000,7 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
if (BranchType == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE) {
if (MemType != MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE)
return IndirectBranchType::UNKNOWN;
- JTType = JumpTable::JTT_PIC;
+ JTType = JumpTable::JTT_X86_64_PIC4;
} else {
if (MemType == MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE)
return IndirectBranchType::UNKNOWN;
@@ -1007,7 +1009,7 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
return IndirectBranchType::POSSIBLE_TAIL_CALL;
BranchType = IndirectBranchType::POSSIBLE_JUMP_TABLE;
- JTType = JumpTable::JTT_NORMAL;
+ JTType = JumpTable::JTT_X86_64_ABS;
}
// Convert the instruction into jump table branch.
@@ -1779,7 +1781,8 @@ void BinaryFunction::postProcessJumpTables() {
// Create labels for all entries.
for (auto &JTI : JumpTables) {
JumpTable &JT = *JTI.second;
- if (JT.Type == JumpTable::JTT_PIC && opts::JumpTables == JTS_BASIC) {
+ if (JT.Type == JumpTable::JTT_X86_64_PIC4 &&
+ opts::JumpTables == JTS_BASIC) {
opts::JumpTables = JTS_MOVE;
BC.outs() << "BOLT-INFO: forcing -jump-tables=move as PIC jump table was "
"detected in function "
@@ -1974,7 +1977,7 @@ bool BinaryFunction::postProcessIndirectBranches(
BC.MIB->unsetJumpTable(Instr);
JumpTable *JT = BC.getJumpTableContainingAddress(LastJT);
- if (JT->Type == JumpTable::JTT_NORMAL) {
+ if (JT->Type == JumpTable::JTT_X86_64_ABS) {
// Invalidating the jump table may also invalidate other jump table
// boundaries. Until we have/need a support for this, mark the
// function as non-simple.
diff --git a/bolt/lib/Core/JumpTable.cpp b/bolt/lib/Core/JumpTable.cpp
index 65e1032..e780c73 100644
--- a/bolt/lib/Core/JumpTable.cpp
+++ b/bolt/lib/Core/JumpTable.cpp
@@ -85,9 +85,9 @@ void bolt::JumpTable::updateOriginal() {
uint64_t EntryOffset = BaseOffset;
for (MCSymbol *Entry : Entries) {
const uint64_t RelType =
- Type == JTT_NORMAL ? ELF::R_X86_64_64 : ELF::R_X86_64_PC32;
+ Type == JTT_X86_64_ABS ? ELF::R_X86_64_64 : ELF::R_X86_64_PC32;
const uint64_t RelAddend =
- Type == JTT_NORMAL ? 0 : EntryOffset - BaseOffset;
+ Type == JTT_X86_64_ABS ? 0 : EntryOffset - BaseOffset;
// Replace existing relocation with the new one to allow any modifications
// to the original jump table.
if (BC.HasRelocations)
@@ -99,7 +99,7 @@ void bolt::JumpTable::updateOriginal() {
void bolt::JumpTable::print(raw_ostream &OS) const {
uint64_t Offset = 0;
- if (Type == JTT_PIC)
+ if (Type == JTT_X86_64_PIC4)
OS << "PIC ";
ListSeparator LS;
diff --git a/bolt/lib/Passes/IndirectCallPromotion.cpp b/bolt/lib/Passes/IndirectCallPromotion.cpp
index 2b5a591..d70fd0e 100644
--- a/bolt/lib/Passes/IndirectCallPromotion.cpp
+++ b/bolt/lib/Passes/IndirectCallPromotion.cpp
@@ -246,7 +246,7 @@ IndirectCallPromotion::getCallTargets(BinaryBasicBlock &BB,
if (const JumpTable *JT = BF.getJumpTable(Inst)) {
// Don't support PIC jump tables for now
- if (!opts::ICPJumpTablesByTarget && JT->Type == JumpTable::JTT_PIC)
+ if (!opts::ICPJumpTablesByTarget && JT->Type == JumpTable::JTT_X86_64_PIC4)
return Targets;
const Location From(BF.getSymbol());
const std::pair<size_t, size_t> Range =
@@ -256,7 +256,7 @@ IndirectCallPromotion::getCallTargets(BinaryBasicBlock &BB,
const JumpTable::JumpInfo *JI =
JT->Counts.empty() ? &DefaultJI : &JT->Counts[Range.first];
const size_t JIAdj = JT->Counts.empty() ? 0 : 1;
- assert(JT->Type == JumpTable::JTT_PIC ||
+ assert(JT->Type == JumpTable::JTT_X86_64_PIC4 ||
JT->EntrySize == BC.AsmInfo->getCodePointerSize());
for (size_t I = Range.first; I < Range.second; ++I, JI += JIAdj) {
MCSymbol *Entry = JT->Entries[I];
diff --git a/bolt/lib/Passes/JTFootprintReduction.cpp b/bolt/lib/Passes/JTFootprintReduction.cpp
index 71bdbba..13b37dc3 100644
--- a/bolt/lib/Passes/JTFootprintReduction.cpp
+++ b/bolt/lib/Passes/JTFootprintReduction.cpp
@@ -202,7 +202,7 @@ bool JTFootprintReduction::tryOptimizePIC(BinaryContext &BC,
JumpTable->OutputEntrySize = 4;
// DePICify
- JumpTable->Type = JumpTable::JTT_NORMAL;
+ JumpTable->Type = JumpTable::JTT_X86_64_ABS;
BB.replaceInstruction(Inst, NewFrag.begin(), NewFrag.end());
return true;
generated by cgit v1.1 at 2025-09-27 05:52:36 +0000