aboutsummaryrefslogtreecommitdiff
path: root/bolt/lib
diff options
context:
space:
mode:
Diffstat (limited to 'bolt/lib')
-rw-r--r--bolt/lib/Core/BinaryContext.cpp96
-rw-r--r--bolt/lib/Core/BinaryFunction.cpp7
-rw-r--r--bolt/lib/Profile/DataAggregator.cpp23
-rw-r--r--bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp6
4 files changed, 105 insertions, 27 deletions
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 7dded4c..c33540a 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -77,6 +77,11 @@ cl::opt<std::string> CompDirOverride(
"location, which is used with DW_AT_dwo_name to construct a path "
"to *.dwo files."),
cl::Hidden, cl::init(""), cl::cat(BoltCategory));
+
+static cl::opt<bool>
+ FailOnInvalidPadding("fail-on-invalid-padding", cl::Hidden, cl::init(false),
+ cl::desc("treat invalid code padding as error"),
+ cl::ZeroOrMore, cl::cat(BoltCategory));
} // namespace opts
namespace llvm {
@@ -942,8 +947,7 @@ std::string BinaryContext::generateJumpTableName(const BinaryFunction &BF,
}
bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) {
- // FIXME: aarch64 support is missing.
- if (!isX86())
+ if (!isX86() && !isAArch64())
return true;
if (BF.getSize() == BF.getMaxSize())
@@ -973,14 +977,26 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) {
return Offset - StartOffset;
};
- // Skip a sequence of zero bytes.
+ // Skip a sequence of zero bytes. For AArch64 we only skip 4 bytes of zeros
+ // in case the following zeros belong to constant island or veneer.
auto skipZeros = [&]() {
const uint64_t StartOffset = Offset;
- for (; Offset < BF.getMaxSize(); ++Offset)
- if ((*FunctionData)[Offset] != 0)
+ uint64_t CurrentOffset = Offset;
+ for (; CurrentOffset < BF.getMaxSize() &&
+ (!isAArch64() || CurrentOffset < StartOffset + 4);
+ ++CurrentOffset)
+ if ((*FunctionData)[CurrentOffset] != 0)
break;
- return Offset - StartOffset;
+ uint64_t NumZeros = CurrentOffset - StartOffset;
+ if (isAArch64())
+ NumZeros &= ~((uint64_t)0x3);
+
+ if (NumZeros == 0)
+ return false;
+ Offset += NumZeros;
+ InstrAddress += NumZeros;
+ return true;
};
// Accept the whole padding area filled with breakpoints.
@@ -993,6 +1009,8 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) {
// Some functions have a jump to the next function or to the padding area
// inserted after the body.
auto isSkipJump = [&](const MCInst &Instr) {
+ if (!isX86())
+ return false;
uint64_t TargetAddress = 0;
if (MIB->isUnconditionalBranch(Instr) &&
MIB->evaluateBranch(Instr, InstrAddress, InstrSize, TargetAddress)) {
@@ -1004,34 +1022,73 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) {
return false;
};
+ // For veneers that are not already covered by binary functions, only those
+ // that handleAArch64Veneer() can recognize are checked here.
+ auto skipAArch64Veneer = [&]() {
+ if (!isAArch64() || Offset >= BF.getMaxSize())
+ return false;
+ BinaryFunction *BFVeneer = getBinaryFunctionContainingAddress(InstrAddress);
+ if (BFVeneer) {
+ // A binary function may have been created to point to this veneer.
+ Offset += BFVeneer->getSize();
+ assert(Offset <= BF.getMaxSize() &&
+ "AArch64 veneeer goes past the max size of function");
+ InstrAddress += BFVeneer->getSize();
+ return true;
+ }
+ const uint64_t AArch64VeneerSize = 12;
+ if (Offset + AArch64VeneerSize <= BF.getMaxSize() &&
+ handleAArch64Veneer(InstrAddress, /*MatchOnly*/ true)) {
+ Offset += AArch64VeneerSize;
+ InstrAddress += AArch64VeneerSize;
+ this->errs() << "BOLT-WARNING: found unmarked AArch64 veneer at 0x"
+ << Twine::utohexstr(BF.getAddress() + Offset) << '\n';
+ return true;
+ }
+ return false;
+ };
+
+ auto skipAArch64ConstantIsland = [&]() {
+ if (!isAArch64() || Offset >= BF.getMaxSize())
+ return false;
+ uint64_t Size;
+ if (BF.isInConstantIsland(InstrAddress, &Size)) {
+ Offset += Size;
+ InstrAddress += Size;
+ return true;
+ }
+ return false;
+ };
+
// Skip over nops, jumps, and zero padding. Allow interleaving (this happens).
- while (skipInstructions(isNoop) || skipInstructions(isSkipJump) ||
+ // For AArch64 also check veneers and skip constant islands.
+ while (skipAArch64Veneer() || skipAArch64ConstantIsland() ||
+ skipInstructions(isNoop) || skipInstructions(isSkipJump) ||
skipZeros())
;
if (Offset == BF.getMaxSize())
return true;
- if (opts::Verbosity >= 1) {
- this->errs() << "BOLT-WARNING: bad padding at address 0x"
- << Twine::utohexstr(BF.getAddress() + BF.getSize())
- << " starting at offset " << (Offset - BF.getSize())
- << " in function " << BF << '\n'
- << FunctionData->slice(BF.getSize(),
- BF.getMaxSize() - BF.getSize())
- << '\n';
- }
-
+ this->errs() << "BOLT-WARNING: bad padding at address 0x"
+ << Twine::utohexstr(BF.getAddress() + BF.getSize())
+ << " starting at offset " << (Offset - BF.getSize())
+ << " in function " << BF << '\n'
+ << FunctionData->slice(BF.getSize(),
+ BF.getMaxSize() - BF.getSize())
+ << '\n';
return false;
}
void BinaryContext::adjustCodePadding() {
+ uint64_t NumInvalid = 0;
for (auto &BFI : BinaryFunctions) {
BinaryFunction &BF = BFI.second;
if (!shouldEmit(BF))
continue;
if (!hasValidCodePadding(BF)) {
+ NumInvalid++;
if (HasRelocations) {
this->errs() << "BOLT-WARNING: function " << BF
<< " has invalid padding. Ignoring the function\n";
@@ -1041,6 +1098,11 @@ void BinaryContext::adjustCodePadding() {
}
}
}
+ if (NumInvalid && opts::FailOnInvalidPadding) {
+ this->errs() << "BOLT-ERROR: found " << NumInvalid
+ << " instance(s) of invalid code padding\n";
+ exit(1);
+ }
}
MCSymbol *BinaryContext::registerNameAtAddress(StringRef Name, uint64_t Address,
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 4dfd4ba..776f4ae 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -1427,10 +1427,9 @@ Error BinaryFunction::disassemble() {
!(BC.isAArch64() &&
BC.handleAArch64Veneer(TargetAddress, /*MatchOnly*/ true))) {
// Result of __builtin_unreachable().
- LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump past end detected at 0x"
- << Twine::utohexstr(AbsoluteInstrAddr)
- << " in function " << *this
- << " : replacing with nop.\n");
+ errs() << "BOLT-WARNING: jump past end detected at 0x"
+ << Twine::utohexstr(AbsoluteInstrAddr) << " in function "
+ << *this << " : replacing with nop.\n";
BC.MIB->createNoop(Instruction);
if (IsCondBranch) {
// Register branch offset for profile validation.
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 9faccc2..dc3d918 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -45,10 +45,23 @@ using namespace bolt;
namespace opts {
static cl::opt<bool>
- BasicAggregation("nl",
- cl::desc("aggregate basic samples (without brstack info)"),
+ BasicAggregation("basic-events",
+ cl::desc("aggregate basic events (without brstack info)"),
cl::cat(AggregatorCategory));
+static cl::alias BasicAggregationAlias("ba",
+ cl::desc("Alias for --basic-events"),
+ cl::aliasopt(BasicAggregation));
+
+static cl::opt<bool> DeprecatedBasicAggregationNl(
+ "nl", cl::desc("Alias for --basic-events (deprecated. Use --ba)"),
+ cl::cat(AggregatorCategory), cl::ReallyHidden,
+ cl::callback([](const bool &Enabled) {
+ errs()
+ << "BOLT-WARNING: '-nl' is deprecated, please use '--ba' instead.\n";
+ BasicAggregation = Enabled;
+ }));
+
cl::opt<bool> ArmSPE("spe", cl::desc("Enable Arm SPE mode."),
cl::cat(AggregatorCategory));
@@ -1433,7 +1446,7 @@ std::error_code DataAggregator::printLBRHeatMap() {
"Cannot build heatmap.";
} else {
errs() << "HEATMAP-ERROR: no brstack traces detected in profile. "
- "Cannot build heatmap. Use -nl for building heatmap from "
+ "Cannot build heatmap. Use -ba for building heatmap from "
"basic events.\n";
}
exit(1);
@@ -1629,8 +1642,8 @@ std::error_code DataAggregator::parseBranchEvents() {
<< "PERF2BOLT-WARNING: all recorded samples for this binary lack "
"brstack. Record profile with perf record -j any or run "
"perf2bolt "
- "in non-brstack mode with -nl (the performance improvement in "
- "-nl "
+ "in non-brstack mode with -ba (the performance improvement in "
+ "-ba "
"mode may be limited)\n";
else
errs()
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index 6954cb2..7769162 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -157,6 +157,10 @@ public:
MCPhysReg getStackPointer() const override { return AArch64::SP; }
MCPhysReg getFramePointer() const override { return AArch64::FP; }
+ bool isBreakpoint(const MCInst &Inst) const override {
+ return Inst.getOpcode() == AArch64::BRK;
+ }
+
bool isPush(const MCInst &Inst) const override {
return isStoreToStack(Inst);
};
@@ -2153,7 +2157,7 @@ public:
--I;
Address -= 4;
- if (I->getOpcode() != AArch64::ADRP ||
+ if (I != Begin || I->getOpcode() != AArch64::ADRP ||
MCPlus::getNumPrimeOperands(*I) < 2 || !I->getOperand(0).isReg() ||
!I->getOperand(1).isImm() || I->getOperand(0).getReg() != AArch64::X16)
return 0;
generated by cgit v1.1 at 2025-10-23 22:00:20 +0000