aboutsummaryrefslogtreecommitdiff
path: root/bolt/lib
diff options
context:
space:
mode:
Diffstat (limited to 'bolt/lib')
-rw-r--r--bolt/lib/Core/BinaryBasicBlock.cpp6
-rw-r--r--bolt/lib/Core/BinaryContext.cpp3
-rw-r--r--bolt/lib/Core/BinaryFunction.cpp25
-rw-r--r--bolt/lib/Core/Exceptions.cpp36
-rw-r--r--bolt/lib/Core/MCPlusBuilder.cpp49
-rw-r--r--bolt/lib/Passes/CMakeLists.txt2
-rw-r--r--bolt/lib/Passes/InsertNegateRAStatePass.cpp142
-rw-r--r--bolt/lib/Passes/MarkRAStates.cpp152
-rw-r--r--bolt/lib/Rewrite/BinaryPassManager.cpp13
-rw-r--r--bolt/lib/Rewrite/RewriteInstance.cpp24
-rw-r--r--bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp22
11 files changed, 443 insertions, 31 deletions
diff --git a/bolt/lib/Core/BinaryBasicBlock.cpp b/bolt/lib/Core/BinaryBasicBlock.cpp
index eeab1ed..d680850 100644
--- a/bolt/lib/Core/BinaryBasicBlock.cpp
+++ b/bolt/lib/Core/BinaryBasicBlock.cpp
@@ -210,7 +210,11 @@ int32_t BinaryBasicBlock::getCFIStateAtInstr(const MCInst *Instr) const {
InstrSeen = (&Inst == Instr);
continue;
}
- if (Function->getBinaryContext().MIB->isCFI(Inst)) {
+ // Ignoring OpNegateRAState CFIs here, as they dont have a "State"
+ // number associated with them.
+ if (Function->getBinaryContext().MIB->isCFI(Inst) &&
+ (Function->getCFIFor(Inst)->getOperation() !=
+ MCCFIInstruction::OpNegateRAState)) {
LastCFI = &Inst;
break;
}
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index b7ded6b..206d8eef 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -1905,6 +1905,9 @@ void BinaryContext::printCFI(raw_ostream &OS, const MCCFIInstruction &Inst) {
case MCCFIInstruction::OpGnuArgsSize:
OS << "OpGnuArgsSize";
break;
+ case MCCFIInstruction::OpNegateRAState:
+ OS << "OpNegateRAState";
+ break;
default:
OS << "Op#" << Operation;
break;
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 07bc71e..9687892 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -2814,14 +2814,8 @@ private:
case MCCFIInstruction::OpLLVMDefAspaceCfa:
case MCCFIInstruction::OpLabel:
case MCCFIInstruction::OpValOffset:
- llvm_unreachable("unsupported CFI opcode");
- break;
case MCCFIInstruction::OpNegateRAState:
- if (!(opts::BinaryAnalysisMode || opts::HeatmapMode)) {
- llvm_unreachable("BOLT-ERROR: binaries using pac-ret hardening (e.g. "
- "as produced by '-mbranch-protection=pac-ret') are "
- "currently not supported by BOLT.");
- }
+ llvm_unreachable("unsupported CFI opcode");
break;
case MCCFIInstruction::OpRememberState:
case MCCFIInstruction::OpRestoreState:
@@ -2836,6 +2830,7 @@ public:
void advanceTo(int32_t State) {
for (int32_t I = CurState, E = State; I != E; ++I) {
const MCCFIInstruction &Instr = FDE[I];
+ assert(Instr.getOperation() != MCCFIInstruction::OpNegateRAState);
if (Instr.getOperation() != MCCFIInstruction::OpRestoreState) {
update(Instr, I);
continue;
@@ -2960,15 +2955,9 @@ struct CFISnapshotDiff : public CFISnapshot {
case MCCFIInstruction::OpLLVMDefAspaceCfa:
case MCCFIInstruction::OpLabel:
case MCCFIInstruction::OpValOffset:
+ case MCCFIInstruction::OpNegateRAState:
llvm_unreachable("unsupported CFI opcode");
return false;
- case MCCFIInstruction::OpNegateRAState:
- if (!(opts::BinaryAnalysisMode || opts::HeatmapMode)) {
- llvm_unreachable("BOLT-ERROR: binaries using pac-ret hardening (e.g. "
- "as produced by '-mbranch-protection=pac-ret') are "
- "currently not supported by BOLT.");
- }
- break;
case MCCFIInstruction::OpRememberState:
case MCCFIInstruction::OpRestoreState:
case MCCFIInstruction::OpGnuArgsSize:
@@ -3117,14 +3106,8 @@ BinaryFunction::unwindCFIState(int32_t FromState, int32_t ToState,
case MCCFIInstruction::OpLLVMDefAspaceCfa:
case MCCFIInstruction::OpLabel:
case MCCFIInstruction::OpValOffset:
- llvm_unreachable("unsupported CFI opcode");
- break;
case MCCFIInstruction::OpNegateRAState:
- if (!(opts::BinaryAnalysisMode || opts::HeatmapMode)) {
- llvm_unreachable("BOLT-ERROR: binaries using pac-ret hardening (e.g. "
- "as produced by '-mbranch-protection=pac-ret') are "
- "currently not supported by BOLT.");
- }
+ llvm_unreachable("unsupported CFI opcode");
break;
case MCCFIInstruction::OpGnuArgsSize:
// do not affect CFI state
diff --git a/bolt/lib/Core/Exceptions.cpp b/bolt/lib/Core/Exceptions.cpp
index 874419f..27656c7 100644
--- a/bolt/lib/Core/Exceptions.cpp
+++ b/bolt/lib/Core/Exceptions.cpp
@@ -568,10 +568,25 @@ bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const {
case DW_CFA_remember_state:
Function.addCFIInstruction(
Offset, MCCFIInstruction::createRememberState(nullptr));
+
+ if (Function.getBinaryContext().isAArch64()) {
+ // Support for pointer authentication:
+ // We need to annotate instructions that modify the RA State, to work
+ // out the state of each instruction in MarkRAStates Pass.
+ if (Offset != 0)
+ Function.setInstModifiesRAState(DW_CFA_remember_state, Offset);
+ }
break;
case DW_CFA_restore_state:
Function.addCFIInstruction(Offset,
MCCFIInstruction::createRestoreState(nullptr));
+ if (Function.getBinaryContext().isAArch64()) {
+ // Support for pointer authentication:
+ // We need to annotate instructions that modify the RA State, to work
+ // out the state of each instruction in MarkRAStates Pass.
+ if (Offset != 0)
+ Function.setInstModifiesRAState(DW_CFA_restore_state, Offset);
+ }
break;
case DW_CFA_def_cfa:
Function.addCFIInstruction(
@@ -629,11 +644,24 @@ bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const {
BC.errs() << "BOLT-WARNING: DW_CFA_MIPS_advance_loc unimplemented\n";
return false;
case DW_CFA_GNU_window_save:
- // DW_CFA_GNU_window_save and DW_CFA_GNU_NegateRAState just use the same
- // id but mean different things. The latter is used in AArch64.
+ // DW_CFA_GNU_window_save and DW_CFA_AARCH64_negate_ra_state just use the
+ // same id but mean different things. The latter is used in AArch64.
if (Function.getBinaryContext().isAArch64()) {
- Function.addCFIInstruction(
- Offset, MCCFIInstruction::createNegateRAState(nullptr));
+ Function.setContainedNegateRAState();
+ // The location OpNegateRAState CFIs are needed depends on the order of
+ // BasicBlocks, which changes during optimizations. Instead of adding
+ // OpNegateRAState CFIs, an annotation is added to the instruction, to
+ // mark that the instruction modifies the RA State. The actual state for
+ // instructions are worked out in MarkRAStates based on these
+ // annotations.
+ if (Offset != 0)
+ Function.setInstModifiesRAState(DW_CFA_AARCH64_negate_ra_state,
+ Offset);
+ else
+ // We cannot Annotate an instruction at Offset == 0.
+ // Instead, we save the initial (Signed) state, and push it to
+ // MarkRAStates' RAStateStack.
+ Function.setInitialRAState(true);
break;
}
if (opts::Verbosity >= 1)
diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp
index 5247522..e96de80 100644
--- a/bolt/lib/Core/MCPlusBuilder.cpp
+++ b/bolt/lib/Core/MCPlusBuilder.cpp
@@ -159,6 +159,55 @@ bool MCPlusBuilder::isTailCall(const MCInst &Inst) const {
return false;
}
+void MCPlusBuilder::setNegateRAState(MCInst &Inst) const {
+ assert(!hasAnnotation(Inst, MCAnnotation::kNegateState));
+ setAnnotationOpValue(Inst, MCAnnotation::kNegateState, true);
+}
+
+bool MCPlusBuilder::hasNegateRAState(const MCInst &Inst) const {
+ return hasAnnotation(Inst, MCAnnotation::kNegateState);
+}
+
+void MCPlusBuilder::setRememberState(MCInst &Inst) const {
+ assert(!hasAnnotation(Inst, MCAnnotation::kRememberState));
+ setAnnotationOpValue(Inst, MCAnnotation::kRememberState, true);
+}
+
+bool MCPlusBuilder::hasRememberState(const MCInst &Inst) const {
+ return hasAnnotation(Inst, MCAnnotation::kRememberState);
+}
+
+void MCPlusBuilder::setRestoreState(MCInst &Inst) const {
+ assert(!hasAnnotation(Inst, MCAnnotation::kRestoreState));
+ setAnnotationOpValue(Inst, MCAnnotation::kRestoreState, true);
+}
+
+bool MCPlusBuilder::hasRestoreState(const MCInst &Inst) const {
+ return hasAnnotation(Inst, MCAnnotation::kRestoreState);
+}
+
+void MCPlusBuilder::setRASigned(MCInst &Inst) const {
+ assert(!hasAnnotation(Inst, MCAnnotation::kRASigned));
+ setAnnotationOpValue(Inst, MCAnnotation::kRASigned, true);
+}
+
+bool MCPlusBuilder::isRASigned(const MCInst &Inst) const {
+ return hasAnnotation(Inst, MCAnnotation::kRASigned);
+}
+
+void MCPlusBuilder::setRAUnsigned(MCInst &Inst) const {
+ assert(!hasAnnotation(Inst, MCAnnotation::kRAUnsigned));
+ setAnnotationOpValue(Inst, MCAnnotation::kRAUnsigned, true);
+}
+
+bool MCPlusBuilder::isRAUnsigned(const MCInst &Inst) const {
+ return hasAnnotation(Inst, MCAnnotation::kRAUnsigned);
+}
+
+bool MCPlusBuilder::isRAStateUnknown(const MCInst &Inst) const {
+ return !(isRAUnsigned(Inst) || isRASigned(Inst));
+}
+
std::optional<MCLandingPad> MCPlusBuilder::getEHInfo(const MCInst &Inst) const {
if (!isCall(Inst))
return std::nullopt;
diff --git a/bolt/lib/Passes/CMakeLists.txt b/bolt/lib/Passes/CMakeLists.txt
index 77d2bb9..d751951 100644
--- a/bolt/lib/Passes/CMakeLists.txt
+++ b/bolt/lib/Passes/CMakeLists.txt
@@ -17,12 +17,14 @@ add_llvm_library(LLVMBOLTPasses
IdenticalCodeFolding.cpp
IndirectCallPromotion.cpp
Inliner.cpp
+ InsertNegateRAStatePass.cpp
Instrumentation.cpp
JTFootprintReduction.cpp
LongJmp.cpp
LoopInversionPass.cpp
LivenessAnalysis.cpp
MCF.cpp
+ MarkRAStates.cpp
PatchEntries.cpp
PAuthGadgetScanner.cpp
PettisAndHansen.cpp
diff --git a/bolt/lib/Passes/InsertNegateRAStatePass.cpp b/bolt/lib/Passes/InsertNegateRAStatePass.cpp
new file mode 100644
index 0000000..33664e1
--- /dev/null
+++ b/bolt/lib/Passes/InsertNegateRAStatePass.cpp
@@ -0,0 +1,142 @@
+//===- bolt/Passes/InsertNegateRAStatePass.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the InsertNegateRAStatePass class. It inserts
+// OpNegateRAState CFIs to places where the state of two consecutive
+// instructions are different.
+//
+//===----------------------------------------------------------------------===//
+#include "bolt/Passes/InsertNegateRAStatePass.h"
+#include "bolt/Core/BinaryFunction.h"
+#include "bolt/Core/ParallelUtilities.h"
+#include <cstdlib>
+
+using namespace llvm;
+
+namespace llvm {
+namespace bolt {
+
+void InsertNegateRAState::runOnFunction(BinaryFunction &BF) {
+ BinaryContext &BC = BF.getBinaryContext();
+
+ if (BF.getState() == BinaryFunction::State::Empty)
+ return;
+
+ if (BF.getState() != BinaryFunction::State::CFG &&
+ BF.getState() != BinaryFunction::State::CFG_Finalized) {
+ BC.outs() << "BOLT-INFO: no CFG for " << BF.getPrintName()
+ << " in InsertNegateRAStatePass\n";
+ return;
+ }
+
+ inferUnknownStates(BF);
+
+ for (FunctionFragment &FF : BF.getLayout().fragments()) {
+ coverFunctionFragmentStart(BF, FF);
+ bool FirstIter = true;
+ MCInst PrevInst;
+ // As this pass runs after function splitting, we should only check
+ // consecutive instructions inside FunctionFragments.
+ for (BinaryBasicBlock *BB : FF) {
+ for (auto It = BB->begin(); It != BB->end(); ++It) {
+ MCInst &Inst = *It;
+ if (BC.MIB->isCFI(Inst))
+ continue;
+ if (!FirstIter) {
+ // Consecutive instructions with different RAState means we need to
+ // add a OpNegateRAState.
+ if ((BC.MIB->isRASigned(PrevInst) && BC.MIB->isRAUnsigned(Inst)) ||
+ (BC.MIB->isRAUnsigned(PrevInst) && BC.MIB->isRASigned(Inst))) {
+ It = BF.addCFIInstruction(
+ BB, It, MCCFIInstruction::createNegateRAState(nullptr));
+ }
+ } else {
+ FirstIter = false;
+ }
+ PrevInst = *It;
+ }
+ }
+ }
+}
+
+void InsertNegateRAState::coverFunctionFragmentStart(BinaryFunction &BF,
+ FunctionFragment &FF) {
+ BinaryContext &BC = BF.getBinaryContext();
+ if (FF.empty())
+ return;
+ // Find the first BB in the FF which has Instructions.
+ // BOLT can generate empty BBs at function splitting which are only used as
+ // target labels. We should add the negate-ra-state CFI to the first
+ // non-empty BB.
+ auto *FirstNonEmpty =
+ std::find_if(FF.begin(), FF.end(), [](BinaryBasicBlock *BB) {
+ // getFirstNonPseudo returns BB.end() if it does not find any
+ // Instructions.
+ return BB->getFirstNonPseudo() != BB->end();
+ });
+ // If a function is already split in the input, the first FF can also start
+ // with Signed state. This covers that scenario as well.
+ if (BC.MIB->isRASigned(*((*FirstNonEmpty)->begin()))) {
+ BF.addCFIInstruction(*FirstNonEmpty, (*FirstNonEmpty)->begin(),
+ MCCFIInstruction::createNegateRAState(nullptr));
+ }
+}
+
+void InsertNegateRAState::inferUnknownStates(BinaryFunction &BF) {
+ BinaryContext &BC = BF.getBinaryContext();
+ bool FirstIter = true;
+ MCInst PrevInst;
+ for (BinaryBasicBlock &BB : BF) {
+ for (MCInst &Inst : BB) {
+ if (BC.MIB->isCFI(Inst))
+ continue;
+
+ if (!FirstIter && BC.MIB->isRAStateUnknown(Inst)) {
+ if (BC.MIB->isRASigned(PrevInst) || BC.MIB->isPSignOnLR(PrevInst)) {
+ BC.MIB->setRASigned(Inst);
+ } else if (BC.MIB->isRAUnsigned(PrevInst) ||
+ BC.MIB->isPAuthOnLR(PrevInst)) {
+ BC.MIB->setRAUnsigned(Inst);
+ }
+ } else {
+ FirstIter = false;
+ }
+ PrevInst = Inst;
+ }
+ }
+}
+
+Error InsertNegateRAState::runOnFunctions(BinaryContext &BC) {
+ std::atomic<uint64_t> FunctionsModified{0};
+ ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) {
+ FunctionsModified++;
+ runOnFunction(BF);
+ };
+
+ ParallelUtilities::PredicateTy SkipPredicate = [&](const BinaryFunction &BF) {
+ // We can skip functions which did not include negate-ra-state CFIs. This
+ // includes code using pac-ret hardening as well, if the binary is
+ // compiled with `-fno-exceptions -fno-unwind-tables
+ // -fno-asynchronous-unwind-tables`
+ return !BF.containedNegateRAState() || BF.isIgnored();
+ };
+
+ ParallelUtilities::runOnEachFunction(
+ BC, ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFun,
+ SkipPredicate, "InsertNegateRAStatePass");
+
+ BC.outs() << "BOLT-INFO: rewritten pac-ret DWARF info in "
+ << FunctionsModified << " out of " << BC.getBinaryFunctions().size()
+ << " functions "
+ << format("(%.2lf%%).\n", (100.0 * FunctionsModified) /
+ BC.getBinaryFunctions().size());
+ return Error::success();
+}
+
+} // end namespace bolt
+} // end namespace llvm
diff --git a/bolt/lib/Passes/MarkRAStates.cpp b/bolt/lib/Passes/MarkRAStates.cpp
new file mode 100644
index 0000000..2c5ce4a
--- /dev/null
+++ b/bolt/lib/Passes/MarkRAStates.cpp
@@ -0,0 +1,152 @@
+//===- bolt/Passes/MarkRAStates.cpp ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the MarkRAStates class.
+// Three CFIs have an influence on the RA State of an instruction:
+// - NegateRAState flips the RA State,
+// - RememberState pushes the RA State to a stack,
+// - RestoreState pops the RA State from the stack.
+// These are saved as MCAnnotations on instructions they refer to at CFI
+// reading (in CFIReaderWriter::fillCFIInfoFor). In this pass, we can work out
+// the RA State of each instruction, and save it as new MCAnnotations. The new
+// annotations are Signing, Signed, Authenticating and Unsigned. After
+// optimizations, .cfi_negate_ra_state CFIs are added to the places where the
+// state changes in InsertNegateRAStatePass.
+//
+//===----------------------------------------------------------------------===//
+#include "bolt/Passes/MarkRAStates.h"
+#include "bolt/Core/BinaryFunction.h"
+#include "bolt/Core/ParallelUtilities.h"
+#include <cstdlib>
+#include <optional>
+#include <stack>
+
+using namespace llvm;
+
+namespace llvm {
+namespace bolt {
+
+bool MarkRAStates::runOnFunction(BinaryFunction &BF) {
+
+ BinaryContext &BC = BF.getBinaryContext();
+
+ for (const BinaryBasicBlock &BB : BF) {
+ for (const MCInst &Inst : BB) {
+ if ((BC.MIB->isPSignOnLR(Inst) ||
+ (BC.MIB->isPAuthOnLR(Inst) && !BC.MIB->isPAuthAndRet(Inst))) &&
+ !BC.MIB->hasNegateRAState(Inst)) {
+ // Not all functions have .cfi_negate_ra_state in them. But if one does,
+ // we expect psign/pauth instructions to have the hasNegateRAState
+ // annotation.
+ BF.setIgnored();
+ BC.outs() << "BOLT-INFO: inconsistent RAStates in function "
+ << BF.getPrintName()
+ << ": ptr sign/auth inst without .cfi_negate_ra_state\n";
+ return false;
+ }
+ }
+ }
+
+ bool RAState = BF.getInitialRAState();
+ std::stack<bool> RAStateStack;
+ RAStateStack.push(RAState);
+
+ for (BinaryBasicBlock &BB : BF) {
+ for (MCInst &Inst : BB) {
+ if (BC.MIB->isCFI(Inst))
+ continue;
+
+ if (BC.MIB->isPSignOnLR(Inst)) {
+ if (RAState) {
+ // RA signing instructions should only follow unsigned RA state.
+ BC.outs() << "BOLT-INFO: inconsistent RAStates in function "
+ << BF.getPrintName()
+ << ": ptr signing inst encountered in Signed RA state\n";
+ BF.setIgnored();
+ return false;
+ }
+ // The signing instruction itself is unsigned, the next will be
+ // signed.
+ BC.MIB->setRAUnsigned(Inst);
+ } else if (BC.MIB->isPAuthOnLR(Inst)) {
+ if (!RAState) {
+ // RA authenticating instructions should only follow signed RA state.
+ BC.outs() << "BOLT-INFO: inconsistent RAStates in function "
+ << BF.getPrintName()
+ << ": ptr authenticating inst encountered in Unsigned RA "
+ "state\n";
+ BF.setIgnored();
+ return false;
+ }
+ // The authenticating instruction itself is signed, but the next will be
+ // unsigned.
+ BC.MIB->setRASigned(Inst);
+ } else if (RAState) {
+ BC.MIB->setRASigned(Inst);
+ } else {
+ BC.MIB->setRAUnsigned(Inst);
+ }
+
+ // Updating RAState. All updates are valid from the next instruction.
+ // Because the same instruction can have remember and restore, the order
+ // here is relevant. This is the reason to loop over Annotations instead
+ // of just checking each in a predefined order.
+ for (unsigned int Idx = 0; Idx < Inst.getNumOperands(); Idx++) {
+ std::optional<int64_t> Annotation =
+ BC.MIB->getAnnotationAtOpIndex(Inst, Idx);
+ if (!Annotation)
+ continue;
+ if (Annotation == MCPlus::MCAnnotation::kNegateState)
+ RAState = !RAState;
+ else if (Annotation == MCPlus::MCAnnotation::kRememberState)
+ RAStateStack.push(RAState);
+ else if (Annotation == MCPlus::MCAnnotation::kRestoreState) {
+ RAState = RAStateStack.top();
+ RAStateStack.pop();
+ }
+ }
+ }
+ }
+ return true;
+}
+
+Error MarkRAStates::runOnFunctions(BinaryContext &BC) {
+ std::atomic<uint64_t> FunctionsIgnored{0};
+ ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) {
+ if (!runOnFunction(BF)) {
+ FunctionsIgnored++;
+ }
+ };
+
+ ParallelUtilities::PredicateTy SkipPredicate = [&](const BinaryFunction &BF) {
+ // We can skip functions which did not include negate-ra-state CFIs. This
+ // includes code using pac-ret hardening as well, if the binary is
+ // compiled with `-fno-exceptions -fno-unwind-tables
+ // -fno-asynchronous-unwind-tables`
+ return !BF.containedNegateRAState() || BF.isIgnored();
+ };
+
+ int Total = llvm::count_if(
+ BC.getBinaryFunctions(),
+ [&](std::pair<const unsigned long, BinaryFunction> &P) {
+ return P.second.containedNegateRAState() && !P.second.isIgnored();
+ });
+
+ ParallelUtilities::runOnEachFunction(
+ BC, ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFun,
+ SkipPredicate, "MarkRAStates");
+ BC.outs() << "BOLT-INFO: MarkRAStates ran on " << Total
+ << " functions. Ignored " << FunctionsIgnored << " functions "
+ << format("(%.2lf%%)", (100.0 * FunctionsIgnored) / Total)
+ << " because of CFI inconsistencies\n";
+
+ return Error::success();
+}
+
+} // end namespace bolt
+} // end namespace llvm
diff --git a/bolt/lib/Rewrite/BinaryPassManager.cpp b/bolt/lib/Rewrite/BinaryPassManager.cpp
index d9b7a2bd..782137e 100644
--- a/bolt/lib/Rewrite/BinaryPassManager.cpp
+++ b/bolt/lib/Rewrite/BinaryPassManager.cpp
@@ -19,11 +19,13 @@
#include "bolt/Passes/IdenticalCodeFolding.h"
#include "bolt/Passes/IndirectCallPromotion.h"
#include "bolt/Passes/Inliner.h"
+#include "bolt/Passes/InsertNegateRAStatePass.h"
#include "bolt/Passes/Instrumentation.h"
#include "bolt/Passes/JTFootprintReduction.h"
#include "bolt/Passes/LongJmp.h"
#include "bolt/Passes/LoopInversionPass.h"
#include "bolt/Passes/MCF.h"
+#include "bolt/Passes/MarkRAStates.h"
#include "bolt/Passes/PLTCall.h"
#include "bolt/Passes/PatchEntries.h"
#include "bolt/Passes/ProfileQualityStats.h"
@@ -276,6 +278,12 @@ static cl::opt<bool> ShortenInstructions("shorten-instructions",
cl::desc("shorten instructions"),
cl::init(true),
cl::cat(BoltOptCategory));
+
+cl::opt<bool>
+ UpdateBranchProtection("update-branch-protection",
+ cl::desc("Rewrites pac-ret DWARF CFI instructions "
+ "(AArch64-only, on by default)"),
+ cl::init(true), cl::Hidden, cl::cat(BoltCategory));
} // namespace opts
namespace llvm {
@@ -353,6 +361,9 @@ Error BinaryFunctionPassManager::runPasses() {
Error BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
BinaryFunctionPassManager Manager(BC);
+ if (BC.isAArch64())
+ Manager.registerPass(std::make_unique<MarkRAStates>());
+
Manager.registerPass(
std::make_unique<EstimateEdgeCounts>(PrintEstimateEdgeCounts));
@@ -512,6 +523,8 @@ Error BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
// targets. No extra instructions after this pass, otherwise we may have
// relocations out of range and crash during linking.
Manager.registerPass(std::make_unique<LongJmpPass>(PrintLongJmp));
+
+ Manager.registerPass(std::make_unique<InsertNegateRAState>());
}
// This pass should always run last.*
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index bfd03e0..c428828 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -917,9 +917,6 @@ void RewriteInstance::discoverFileObjects() {
bool IsData = false;
uint64_t LastAddr = 0;
for (const auto &SymInfo : SortedSymbols) {
- if (LastAddr == SymInfo.Address) // don't repeat markers
- continue;
-
MarkerSymType MarkerType = BC->getMarkerType(SymInfo.Symbol);
// Treat ST_Function as code.
@@ -929,8 +926,14 @@ void RewriteInstance::discoverFileObjects() {
if (IsData) {
Expected<StringRef> NameOrError = SymInfo.Symbol.getName();
consumeError(NameOrError.takeError());
- BC->errs() << "BOLT-WARNING: function symbol " << *NameOrError
- << " lacks code marker\n";
+ if (LastAddr == SymInfo.Address) {
+ BC->errs() << "BOLT-WARNING: ignoring data marker conflicting with "
+ "function symbol "
+ << *NameOrError << '\n';
+ } else {
+ BC->errs() << "BOLT-WARNING: function symbol " << *NameOrError
+ << " lacks code marker\n";
+ }
}
MarkerType = MarkerSymType::CODE;
}
@@ -3521,6 +3524,17 @@ void RewriteInstance::disassembleFunctions() {
}
}
+ // Check if fillCFIInfoFor removed any OpNegateRAState CFIs from the
+ // function.
+ if (Function.containedNegateRAState()) {
+ if (!opts::UpdateBranchProtection) {
+ BC->errs()
+ << "BOLT-ERROR: --update-branch-protection is set to false, but "
+ << Function.getPrintName() << " contains .cfi-negate-ra-state\n";
+ exit(1);
+ }
+ }
+
// Parse LSDA.
if (Function.getLSDAAddress() != 0 &&
!BC->getFragmentsToSkip().count(&Function)) {
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index f271867..df4f421 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -244,6 +244,28 @@ public:
}
}
+ bool isPSignOnLR(const MCInst &Inst) const override {
+ std::optional<MCPhysReg> SignReg = getSignedReg(Inst);
+ return SignReg && *SignReg == AArch64::LR;
+ }
+
+ bool isPAuthOnLR(const MCInst &Inst) const override {
+ // LDR(A|B) should not be covered.
+ bool IsChecked;
+ std::optional<MCPhysReg> AuthReg =
+ getWrittenAuthenticatedReg(Inst, IsChecked);
+ return !IsChecked && AuthReg && *AuthReg == AArch64::LR;
+ }
+
+ bool isPAuthAndRet(const MCInst &Inst) const override {
+ return Inst.getOpcode() == AArch64::RETAA ||
+ Inst.getOpcode() == AArch64::RETAB ||
+ Inst.getOpcode() == AArch64::RETAASPPCi ||
+ Inst.getOpcode() == AArch64::RETABSPPCi ||
+ Inst.getOpcode() == AArch64::RETAASPPCr ||
+ Inst.getOpcode() == AArch64::RETABSPPCr;
+ }
+
std::optional<MCPhysReg> getSignedReg(const MCInst &Inst) const override {
switch (Inst.getOpcode()) {
case AArch64::PACIA:
generated by cgit v1.1 at 2025-10-07 13:08:06 +0000