[𝘀𝗽𝗿] changes to main this commit is based onusers/aaupov/spr/main.boltnfc-pre-disasm-metadata-rewriters

Created using spr 1.3.4

[skip ci]

8 files changed, 119 insertions, 96 deletions

diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 485979f..69b8f02 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -435,7 +435,18 @@ public:
 
   /// Return size of an entry for the given jump table \p Type.
   uint64_t getJumpTableEntrySize(JumpTable::JumpTableType Type) const {
-    return Type == JumpTable::JTT_PIC ? 4 : AsmInfo->getCodePointerSize();
+    switch (Type) {
+    case JumpTable::JTT_X86_64_PIC4:
+      return 4;
+    case JumpTable::JTT_X86_64_ABS:
+      return AsmInfo->getCodePointerSize();
+    case JumpTable::JTT_AARCH64_REL1:
+      return 1;
+    case JumpTable::JTT_AARCH64_REL2:
+      return 2;
+    case JumpTable::JTT_AARCH64_REL4:
+      return 4;
+    }
   }
 
   /// Return JumpTable containing a given \p Address.
@@ -573,14 +584,13 @@ public:
   /// If \p NextJTAddress is different from zero, it is used as an upper
   /// bound for jump table memory layout.
   ///
-  /// Optionally, populate \p Address from jump table entries. The entries
-  /// could be partially populated if the jump table detection fails.
+  /// If \p JT is set, populate it with jump table entries. The entries could be
+  /// partially populated if the jump table detection fails.
   bool analyzeJumpTable(const uint64_t Address,
                         const JumpTable::JumpTableType Type,
                         const BinaryFunction &BF,
                         const uint64_t NextJTAddress = 0,
-                        JumpTable::AddressesType *EntriesAsAddress = nullptr,
-                        bool *HasEntryInFragment = nullptr) const;
+                        JumpTable *JT = nullptr) const;
 
   /// After jump table locations are established, this function will populate
   /// their EntriesAsAddress based on memory contents.
diff --git a/bolt/include/bolt/Core/JumpTable.h b/bolt/include/bolt/Core/JumpTable.h
index 52b9cce..7dc0c65 100644
--- a/bolt/include/bolt/Core/JumpTable.h
+++ b/bolt/include/bolt/Core/JumpTable.h
@@ -47,10 +47,34 @@ class JumpTable : public BinaryData {
 
 public:
   enum JumpTableType : char {
-    JTT_NORMAL,
-    JTT_PIC,
+    JTT_X86_64_FIRST = 0,
+    JTT_X86_64_ABS = JTT_X86_64_FIRST,
+    JTT_X86_64_PIC4,
+    JTT_X86_64_LAST = JTT_X86_64_PIC4,
+    JTT_AARCH64_FIRST,
+    JTT_AARCH64_REL1 = JTT_AARCH64_FIRST,
+    JTT_AARCH64_REL2,
+    JTT_AARCH64_REL4,
+    JTT_AARCH64_LAST = JTT_AARCH64_REL4
   };
 
+  static StringRef getTypeStr(JumpTableType Type) {
+    switch (Type) {
+    case JTT_X86_64_ABS:
+      return "X86_64_ABS";
+    case JTT_X86_64_PIC4:
+      return "X86_64_PIC4";
+    case JTT_AARCH64_REL1:
+      return "AARCH64_REL1";
+    case JTT_AARCH64_REL2:
+      return "AARCH64_REL2";
+    case JTT_AARCH64_REL4:
+      return "AARCH64_REL4";
+    }
+  }
+
+  const StringRef getTypeStr() { return getTypeStr(Type); }
+
   /// Branch statistics for jump table entries.
   struct JumpInfo {
     uint64_t Mispreds{0};
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 8fa1e36..cf214ac 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -496,7 +496,7 @@ BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF,
     const MemoryContentsType MemType = analyzeMemoryAt(Address, BF);
     if (MemType == MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE && IsPCRel) {
       const MCSymbol *Symbol =
-          getOrCreateJumpTable(BF, Address, JumpTable::JTT_PIC);
+          getOrCreateJumpTable(BF, Address, JumpTable::JTT_X86_64_PIC4);
 
       return std::make_pair(Symbol, 0);
     }
@@ -540,10 +540,10 @@ MemoryContentsType BinaryContext::analyzeMemoryAt(uint64_t Address,
 
   // Start with checking for PIC jump table. We expect non-PIC jump tables
   // to have high 32 bits set to 0.
-  if (analyzeJumpTable(Address, JumpTable::JTT_PIC, BF))
+  if (analyzeJumpTable(Address, JumpTable::JTT_X86_64_PIC4, BF))
     return MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE;
 
-  if (analyzeJumpTable(Address, JumpTable::JTT_NORMAL, BF))
+  if (analyzeJumpTable(Address, JumpTable::JTT_X86_64_ABS, BF))
     return MemoryContentsType::POSSIBLE_JUMP_TABLE;
 
   return MemoryContentsType::UNKNOWN;
@@ -553,8 +553,7 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
                                      const JumpTable::JumpTableType Type,
                                      const BinaryFunction &BF,
                                      const uint64_t NextJTAddress,
-                                     JumpTable::AddressesType *EntriesAsAddress,
-                                     bool *HasEntryInFragment) const {
+                                     JumpTable *JT) const {
   // Target address of __builtin_unreachable.
   const uint64_t UnreachableAddress = BF.getAddress() + BF.getSize();
 
@@ -571,11 +570,11 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
   size_t TrimmedSize = 0;
 
   auto addEntryAddress = [&](uint64_t EntryAddress, bool Unreachable = false) {
-    if (!EntriesAsAddress)
+    if (!JT)
       return;
-    EntriesAsAddress->emplace_back(EntryAddress);
+    JT->EntriesAsAddress.emplace_back(EntryAddress);
     if (!Unreachable)
-      TrimmedSize = EntriesAsAddress->size();
+      TrimmedSize = JT->EntriesAsAddress.size();
   };
 
   ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);
@@ -594,12 +593,9 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
   if (NextJTAddress)
     UpperBound = std::min(NextJTAddress, UpperBound);
 
-  LLVM_DEBUG({
-    using JTT = JumpTable::JumpTableType;
-    dbgs() << formatv("BOLT-DEBUG: analyzeJumpTable @{0:x} in {1}, JTT={2}\n",
-                      Address, BF.getPrintName(),
-                      Type == JTT::JTT_PIC ? "PIC" : "Normal");
-  });
+  LLVM_DEBUG(
+      dbgs() << formatv("BOLT-DEBUG: analyzeJumpTable @{0:x} in {1}, JTT={2}\n",
+                        Address, BF, JumpTable::getTypeStr(Type)));
   const uint64_t EntrySize = getJumpTableEntrySize(Type);
   for (uint64_t EntryAddress = Address; EntryAddress <= UpperBound - EntrySize;
        EntryAddress += EntrySize) {
@@ -607,13 +603,13 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
                       << " -> ");
     // Check if there's a proper relocation against the jump table entry.
     if (HasRelocations) {
-      if (Type == JumpTable::JTT_PIC &&
+      if (Type == JumpTable::JTT_X86_64_PIC4 &&
           !DataPCRelocations.count(EntryAddress)) {
         LLVM_DEBUG(
             dbgs() << "FAIL: JTT_PIC table, no relocation for this address\n");
         break;
       }
-      if (Type == JumpTable::JTT_NORMAL && !getRelocationAt(EntryAddress)) {
+      if (Type == JumpTable::JTT_X86_64_ABS && !getRelocationAt(EntryAddress)) {
         LLVM_DEBUG(
             dbgs()
             << "FAIL: JTT_NORMAL table, no relocation for this address\n");
@@ -621,10 +617,15 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
       }
     }
 
-    const uint64_t Value =
-        (Type == JumpTable::JTT_PIC)
-            ? Address + *getSignedValueAtAddress(EntryAddress, EntrySize)
-            : *getPointerAtAddress(EntryAddress);
+    uint64_t Value = 0;
+    switch (Type) {
+    case JumpTable::JTT_X86_64_PIC4:
+      Value = Address + *getSignedValueAtAddress(EntryAddress, EntrySize);
+      break;
+    case JumpTable::JTT_X86_64_ABS:
+      Value = *getPointerAtAddress(EntryAddress);
+      break;
+    }
 
     // __builtin_unreachable() case.
     if (Value == UnreachableAddress) {
@@ -645,24 +646,19 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
 
     // Function or one of its fragments.
     const BinaryFunction *TargetBF = getBinaryFunctionContainingAddress(Value);
-    const bool DoesBelongToFunction =
-        BF.containsAddress(Value) ||
-        (TargetBF && areRelatedFragments(TargetBF, &BF));
-    if (!DoesBelongToFunction) {
+    if (!TargetBF || !areRelatedFragments(TargetBF, &BF)) {
       LLVM_DEBUG({
-        if (!BF.containsAddress(Value)) {
-          dbgs() << "FAIL: function doesn't contain this address\n";
-          if (TargetBF) {
-            dbgs() << "  ! function containing this address: "
-                   << TargetBF->getPrintName() << '\n';
-            if (TargetBF->isFragment()) {
-              dbgs() << "  ! is a fragment";
-              for (BinaryFunction *Parent : TargetBF->ParentFragments)
-                dbgs() << ", parent: " << Parent->getPrintName();
-              dbgs() << '\n';
-            }
-          }
-        }
+        dbgs() << "FAIL: function doesn't contain this address\n";
+        if (!TargetBF)
+          break;
+        dbgs() << "  ! function containing this address: " << *TargetBF << '\n';
+        if (!TargetBF->isFragment())
+          break;
+        dbgs() << "  ! is a fragment with parents: ";
+        ListSeparator LS;
+        for (BinaryFunction *Parent : TargetBF->ParentFragments)
+          dbgs() << LS << *Parent;
+        dbgs() << '\n';
       });
       break;
     }
@@ -677,17 +673,17 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
     ++NumRealEntries;
     LLVM_DEBUG(dbgs() << formatv("OK: {0:x} real entry\n", Value));
 
-    if (TargetBF != &BF && HasEntryInFragment)
-      *HasEntryInFragment = true;
+    if (TargetBF != &BF && JT)
+      JT->IsSplit = true;
     addEntryAddress(Value);
   }
 
   // Trim direct/normal jump table to exclude trailing unreachable entries that
   // can collide with a function address.
-  if (Type == JumpTable::JTT_NORMAL && EntriesAsAddress &&
-      TrimmedSize != EntriesAsAddress->size() &&
+  if (Type == JumpTable::JTT_X86_64_ABS && JT &&
+      TrimmedSize != JT->EntriesAsAddress.size() &&
       getBinaryFunctionAtAddress(UnreachableAddress))
-    EntriesAsAddress->resize(TrimmedSize);
+    JT->EntriesAsAddress.resize(TrimmedSize);
 
   // It's a jump table if the number of real entries is more than 1, or there's
   // one real entry and one or more special targets. If there are only multiple
@@ -702,10 +698,8 @@ void BinaryContext::populateJumpTables() {
        ++JTI) {
     JumpTable *JT = JTI->second;
 
-    bool NonSimpleParent = false;
-    for (BinaryFunction *BF : JT->Parents)
-      NonSimpleParent |= !BF->isSimple();
-    if (NonSimpleParent)
+    auto isSimple = std::bind(&BinaryFunction::isSimple, std::placeholders::_1);
+    if (!llvm::all_of(JT->Parents, isSimple))
       continue;
 
     uint64_t NextJTAddress = 0;
@@ -713,9 +707,8 @@ void BinaryContext::populateJumpTables() {
     if (NextJTI != JTE)
       NextJTAddress = NextJTI->second->getAddress();
 
-    const bool Success =
-        analyzeJumpTable(JT->getAddress(), JT->Type, *(JT->Parents[0]),
-                         NextJTAddress, &JT->EntriesAsAddress, &JT->IsSplit);
+    const bool Success = analyzeJumpTable(
+        JT->getAddress(), JT->Type, *JT->Parents.front(), NextJTAddress, JT);
     if (!Success) {
       LLVM_DEBUG({
         dbgs() << "failed to analyze ";
@@ -743,7 +736,7 @@ void BinaryContext::populateJumpTables() {
 
     // In strict mode, erase PC-relative relocation record. Later we check that
     // all such records are erased and thus have been accounted for.
-    if (opts::StrictMode && JT->Type == JumpTable::JTT_PIC) {
+    if (opts::StrictMode && JT->Type == JumpTable::JTT_X86_64_PIC4) {
       for (uint64_t Address = JT->getAddress();
            Address < JT->getAddress() + JT->getSize();
            Address += JT->EntrySize) {
@@ -839,33 +832,26 @@ BinaryContext::getOrCreateJumpTable(BinaryFunction &Function, uint64_t Address,
     assert(JT->Type == Type && "jump table types have to match");
     assert(Address == JT->getAddress() && "unexpected non-empty jump table");
 
-    // Prevent associating a jump table to a specific fragment twice.
-    if (!llvm::is_contained(JT->Parents, &Function)) {
-      assert(llvm::all_of(JT->Parents,
-                          [&](const BinaryFunction *BF) {
-                            return areRelatedFragments(&Function, BF);
-                          }) &&
-             "cannot re-use jump table of a different function");
-      // Duplicate the entry for the parent function for easy access
-      JT->Parents.push_back(&Function);
-      if (opts::Verbosity > 2) {
-        this->outs() << "BOLT-INFO: Multiple fragments access same jump table: "
-                     << JT->Parents[0]->getPrintName() << "; "
-                     << Function.getPrintName() << "\n";
-        JT->print(this->outs());
-      }
-      Function.JumpTables.emplace(Address, JT);
-      for (BinaryFunction *Parent : JT->Parents)
-        Parent->setHasIndirectTargetToSplitFragment(true);
-    }
+    if (llvm::is_contained(JT->Parents, &Function))
+      return JT->getFirstLabel();
 
-    bool IsJumpTableParent = false;
-    (void)IsJumpTableParent;
-    for (BinaryFunction *Frag : JT->Parents)
-      if (Frag == &Function)
-        IsJumpTableParent = true;
-    assert(IsJumpTableParent &&
+    // Prevent associating a jump table to a specific fragment twice.
+    auto isSibling = std::bind(&BinaryContext::areRelatedFragments, this,
+                               &Function, std::placeholders::_1);
+    assert(llvm::all_of(JT->Parents, isSibling) &&
            "cannot re-use jump table of a different function");
+    if (opts::Verbosity > 2) {
+      this->outs() << "BOLT-INFO: Multiple fragments access same jump table: "
+                   << JT->Parents[0]->getPrintName() << "; "
+                   << Function.getPrintName() << "\n";
+      JT->print(this->outs());
+    }
+    if (JT->Parents.size() == 1)
+      JT->Parents.front()->setHasIndirectTargetToSplitFragment(true);
+    Function.setHasIndirectTargetToSplitFragment(true);
+    // Duplicate the entry for the parent function for easy access
+    JT->Parents.push_back(&Function);
+    Function.JumpTables.emplace(Address, JT);
     return JT->getFirstLabel();
   }
 
diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp
index 1aad252..7727204 100644
--- a/bolt/lib/Core/BinaryEmitter.cpp
+++ b/bolt/lib/Core/BinaryEmitter.cpp
@@ -850,7 +850,7 @@ void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection,
       }
       LastLabel = LI->second;
     }
-    if (JT.Type == JumpTable::JTT_NORMAL) {
+    if (JT.Type == JumpTable::JTT_X86_64_ABS) {
       Streamer.emitSymbolValue(Entry, JT.OutputEntrySize);
     } else { // JTT_PIC
       const MCSymbolRefExpr *JTExpr =
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index b46ba1a..effd3c4 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -912,7 +912,7 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
            "Invalid memory instruction");
     const MCExpr *FixedEntryDispExpr = FixedEntryDispOperand->getExpr();
     const uint64_t EntryAddress = getExprValue(FixedEntryDispExpr);
-    uint64_t EntrySize = BC.getJumpTableEntrySize(JumpTable::JTT_PIC);
+    uint64_t EntrySize = BC.getJumpTableEntrySize(JumpTable::JTT_X86_64_PIC4);
     ErrorOr<int64_t> Value =
         BC.getSignedValueAtAddress(EntryAddress, EntrySize);
     if (!Value)
@@ -982,12 +982,14 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
   MemoryContentsType MemType;
   if (JumpTable *JT = BC.getJumpTableContainingAddress(ArrayStart)) {
     switch (JT->Type) {
-    case JumpTable::JTT_NORMAL:
+    case JumpTable::JTT_X86_64_ABS:
       MemType = MemoryContentsType::POSSIBLE_JUMP_TABLE;
       break;
-    case JumpTable::JTT_PIC:
+    case JumpTable::JTT_X86_64_PIC4:
       MemType = MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE;
       break;
+    default:
+      llvm_unreachable("Unhandled jump table type");
     }
   } else {
     MemType = BC.analyzeMemoryAt(ArrayStart, *this);
@@ -998,7 +1000,7 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
   if (BranchType == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE) {
     if (MemType != MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE)
       return IndirectBranchType::UNKNOWN;
-    JTType = JumpTable::JTT_PIC;
+    JTType = JumpTable::JTT_X86_64_PIC4;
   } else {
     if (MemType == MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE)
       return IndirectBranchType::UNKNOWN;
@@ -1007,7 +1009,7 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
       return IndirectBranchType::POSSIBLE_TAIL_CALL;
 
     BranchType = IndirectBranchType::POSSIBLE_JUMP_TABLE;
-    JTType = JumpTable::JTT_NORMAL;
+    JTType = JumpTable::JTT_X86_64_ABS;
   }
 
   // Convert the instruction into jump table branch.
@@ -1779,7 +1781,8 @@ void BinaryFunction::postProcessJumpTables() {
   // Create labels for all entries.
   for (auto &JTI : JumpTables) {
     JumpTable &JT = *JTI.second;
-    if (JT.Type == JumpTable::JTT_PIC && opts::JumpTables == JTS_BASIC) {
+    if (JT.Type == JumpTable::JTT_X86_64_PIC4 &&
+        opts::JumpTables == JTS_BASIC) {
       opts::JumpTables = JTS_MOVE;
       BC.outs() << "BOLT-INFO: forcing -jump-tables=move as PIC jump table was "
                    "detected in function "
@@ -1974,7 +1977,7 @@ bool BinaryFunction::postProcessIndirectBranches(
           BC.MIB->unsetJumpTable(Instr);
 
           JumpTable *JT = BC.getJumpTableContainingAddress(LastJT);
-          if (JT->Type == JumpTable::JTT_NORMAL) {
+          if (JT->Type == JumpTable::JTT_X86_64_ABS) {
             // Invalidating the jump table may also invalidate other jump table
             // boundaries. Until we have/need a support for this, mark the
             // function as non-simple.
diff --git a/bolt/lib/Core/JumpTable.cpp b/bolt/lib/Core/JumpTable.cpp
index 65e1032..e780c73 100644
--- a/bolt/lib/Core/JumpTable.cpp
+++ b/bolt/lib/Core/JumpTable.cpp
@@ -85,9 +85,9 @@ void bolt::JumpTable::updateOriginal() {
   uint64_t EntryOffset = BaseOffset;
   for (MCSymbol *Entry : Entries) {
     const uint64_t RelType =
-        Type == JTT_NORMAL ? ELF::R_X86_64_64 : ELF::R_X86_64_PC32;
+        Type == JTT_X86_64_ABS ? ELF::R_X86_64_64 : ELF::R_X86_64_PC32;
     const uint64_t RelAddend =
-        Type == JTT_NORMAL ? 0 : EntryOffset - BaseOffset;
+        Type == JTT_X86_64_ABS ? 0 : EntryOffset - BaseOffset;
     // Replace existing relocation with the new one to allow any modifications
     // to the original jump table.
     if (BC.HasRelocations)
@@ -99,7 +99,7 @@ void bolt::JumpTable::updateOriginal() {
 
 void bolt::JumpTable::print(raw_ostream &OS) const {
   uint64_t Offset = 0;
-  if (Type == JTT_PIC)
+  if (Type == JTT_X86_64_PIC4)
     OS << "PIC ";
   ListSeparator LS;
 
diff --git a/bolt/lib/Passes/IndirectCallPromotion.cpp b/bolt/lib/Passes/IndirectCallPromotion.cpp
index 2b5a591..d70fd0e 100644
--- a/bolt/lib/Passes/IndirectCallPromotion.cpp
+++ b/bolt/lib/Passes/IndirectCallPromotion.cpp
@@ -246,7 +246,7 @@ IndirectCallPromotion::getCallTargets(BinaryBasicBlock &BB,
 
   if (const JumpTable *JT = BF.getJumpTable(Inst)) {
     // Don't support PIC jump tables for now
-    if (!opts::ICPJumpTablesByTarget && JT->Type == JumpTable::JTT_PIC)
+    if (!opts::ICPJumpTablesByTarget && JT->Type == JumpTable::JTT_X86_64_PIC4)
       return Targets;
     const Location From(BF.getSymbol());
     const std::pair<size_t, size_t> Range =
@@ -256,7 +256,7 @@ IndirectCallPromotion::getCallTargets(BinaryBasicBlock &BB,
     const JumpTable::JumpInfo *JI =
         JT->Counts.empty() ? &DefaultJI : &JT->Counts[Range.first];
     const size_t JIAdj = JT->Counts.empty() ? 0 : 1;
-    assert(JT->Type == JumpTable::JTT_PIC ||
+    assert(JT->Type == JumpTable::JTT_X86_64_PIC4 ||
            JT->EntrySize == BC.AsmInfo->getCodePointerSize());
     for (size_t I = Range.first; I < Range.second; ++I, JI += JIAdj) {
       MCSymbol *Entry = JT->Entries[I];
diff --git a/bolt/lib/Passes/JTFootprintReduction.cpp b/bolt/lib/Passes/JTFootprintReduction.cpp
index 71bdbba..13b37dc3 100644
--- a/bolt/lib/Passes/JTFootprintReduction.cpp
+++ b/bolt/lib/Passes/JTFootprintReduction.cpp
@@ -202,7 +202,7 @@ bool JTFootprintReduction::tryOptimizePIC(BinaryContext &BC,
 
   JumpTable->OutputEntrySize = 4;
   // DePICify
-  JumpTable->Type = JumpTable::JTT_NORMAL;
+  JumpTable->Type = JumpTable::JTT_X86_64_ABS;
 
   BB.replaceInstruction(Inst, NewFrag.begin(), NewFrag.end());
   return true;