diff options
Diffstat (limited to 'llvm/lib')
15 files changed, 1322 insertions, 149 deletions
diff --git a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt index e5f5a99..65dd0c7 100644 --- a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt @@ -3,6 +3,7 @@ tablegen(LLVM COFFOptions.inc -gen-opt-parser-defs) add_public_tablegen_target(JITLinkTableGen) add_llvm_component_library(LLVMJITLink + CompactUnwindSupport.cpp DWARFRecordSectionSplitter.cpp EHFrameSupport.cpp JITLink.cpp diff --git a/llvm/lib/ExecutionEngine/JITLink/CompactUnwindSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/CompactUnwindSupport.cpp new file mode 100644 index 0000000..51e3d26 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/CompactUnwindSupport.cpp @@ -0,0 +1,103 @@ +//=------- CompactUnwindSupport.cpp - Compact Unwind format support -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Compact Unwind support. +// +//===----------------------------------------------------------------------===// + +#include "CompactUnwindSupport.h" + +#include "llvm/ADT/Sequence.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +Error splitCompactUnwindBlocks(LinkGraph &G, Section &CompactUnwindSection, + size_t RecordSize) { + + std::vector<Block *> OriginalBlocks(CompactUnwindSection.blocks().begin(), + CompactUnwindSection.blocks().end()); + LLVM_DEBUG({ + dbgs() << "In " << G.getName() << " splitting compact unwind section " + << CompactUnwindSection.getName() << " containing " + << OriginalBlocks.size() << " initial blocks...\n"; + }); + + while (!OriginalBlocks.empty()) { + auto *B = OriginalBlocks.back(); + OriginalBlocks.pop_back(); + + if (B->getSize() == 0) { + LLVM_DEBUG({ + dbgs() << " Skipping empty block at " + << formatv("{0:x16}", B->getAddress()) << "\n"; + }); + continue; + } + + unsigned NumBlocks = B->getSize() / RecordSize; + + LLVM_DEBUG({ + dbgs() << " Splitting block at " << formatv("{0:x16}", B->getAddress()) + << " into " << NumBlocks << " compact unwind record(s)\n"; + }); + + if (B->getSize() % RecordSize) + return make_error<JITLinkError>( + "Error splitting compact unwind record in " + G.getName() + + ": block at " + formatv("{0:x}", B->getAddress()) + " has size " + + formatv("{0:x}", B->getSize()) + + " (not a multiple of CU record size of " + + formatv("{0:x}", RecordSize) + ")"); + + auto Blocks = + G.splitBlock(*B, map_range(seq(1U, NumBlocks), [=](Edge::OffsetT Idx) { + return Idx * RecordSize; + })); + + for (auto *CURec : Blocks) { + bool AddedKeepAlive = false; + + for (auto &E : CURec->edges()) { + if (E.getOffset() == 0) { + LLVM_DEBUG({ + dbgs() << " Updating compact unwind record at " + << CURec->getAddress() << " to point to " + << (E.getTarget().hasName() ? *E.getTarget().getName() + : StringRef()) + << " (at " << E.getTarget().getAddress() << ")\n"; + }); + + if (E.getTarget().isExternal()) + return make_error<JITLinkError>( + "Error adding keep-alive edge for compact unwind record at " + + formatv("{0:x}", CURec->getAddress()) + ": target " + + *E.getTarget().getName() + " is an external symbol"); + auto &TgtBlock = E.getTarget().getBlock(); + auto &CURecSym = + G.addAnonymousSymbol(*CURec, 0, RecordSize, false, false); + TgtBlock.addEdge(Edge::KeepAlive, 0, CURecSym, 0); + AddedKeepAlive = true; + } + } + + if (!AddedKeepAlive) + return make_error<JITLinkError>( + "Error adding keep-alive edge for compact unwind record at " + + formatv("{0:x}", CURec->getAddress()) + + ": no outgoing target edge at offset 0"); + } + } + + return Error::success(); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/CompactUnwindSupport.h b/llvm/lib/ExecutionEngine/JITLink/CompactUnwindSupport.h new file mode 100644 index 0000000..dc3ed94 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/CompactUnwindSupport.h @@ -0,0 +1,653 @@ +//===- CompactUnwindSupportImpl.h - Compact Unwind format impl --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Compact Unwind format support implementation details. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_COMPACTUNWINDSUPPORTIMPL_H +#define LIB_EXECUTIONENGINE_JITLINK_COMPACTUNWINDSUPPORTIMPL_H + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ExecutionEngine/JITLink/MachO.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Endian.h" + +#define DEBUG_TYPE "jitlink_cu" + +namespace llvm { +namespace jitlink { + +/// Split blocks in an __LD,__compact_unwind section on record boundaries. +/// When this function returns edges within each record are guaranteed to be +/// sorted by offset. +Error splitCompactUnwindBlocks(LinkGraph &G, Section &CompactUnwindSection, + size_t RecordSize); + +/// CRTP base for compact unwind traits classes. Automatically provides derived +/// constants. +/// +/// FIXME: Passing PtrSize as a template parameter is a hack to work around a +/// bug in older MSVC compilers (until at least MSVC 15) where constexpr +/// fields in the CRTP impl class were not visible to the base class. +/// Once we no longer need to support these compilers the PtrSize +/// template argument should be removed and PointerSize should be +/// defined as a member in the CRTP Impl classes. +template <typename CRTPImpl, size_t PtrSize> struct CompactUnwindTraits { + static constexpr size_t PointerSize = PtrSize; + static constexpr size_t Size = 3 * PointerSize + 2 * 4; + static constexpr size_t FnFieldOffset = 0; + static constexpr size_t SizeFieldOffset = FnFieldOffset + PointerSize; + static constexpr size_t EncodingFieldOffset = SizeFieldOffset + 4; + static constexpr size_t PersonalityFieldOffset = EncodingFieldOffset + 4; + static constexpr size_t LSDAFieldOffset = + PersonalityFieldOffset + PointerSize; + + static uint32_t readPCRangeSize(ArrayRef<char> RecordContent) { + assert(SizeFieldOffset + 4 <= RecordContent.size() && + "Truncated CU record?"); + return support::endian::read32<CRTPImpl::Endianness>(RecordContent.data() + + SizeFieldOffset); + } + + static uint32_t readEncoding(ArrayRef<char> RecordContent) { + assert(EncodingFieldOffset + 4 <= RecordContent.size() && + "Truncated CU record?"); + return support::endian::read32<CRTPImpl::Endianness>(RecordContent.data() + + EncodingFieldOffset); + } +}; + +/// Architecture specific implementation of CompactUnwindManager. +template <typename CURecTraits> class CompactUnwindManager { +public: + CompactUnwindManager(StringRef CompactUnwindSectionName, + StringRef UnwindInfoSectionName, + StringRef EHFrameSectionName) + : CompactUnwindSectionName(CompactUnwindSectionName), + UnwindInfoSectionName(UnwindInfoSectionName), + EHFrameSectionName(EHFrameSectionName) {} + + // Split compact unwind records, add keep-alive edges from functions to + // compact unwind records, and from compact unwind records to FDEs where + // needed. + // + // This method must be called *after* __eh_frame has been processed: it + // assumes that eh-frame records have been split up and keep-alive edges have + // been inserted. + Error prepareForPrune(LinkGraph &G) { + Section *CUSec = G.findSectionByName(CompactUnwindSectionName); + if (!CUSec || CUSec->empty()) { + LLVM_DEBUG({ + dbgs() << "Compact unwind: No compact unwind info for " << G.getName() + << "\n"; + }); + return Error::success(); + } + + LLVM_DEBUG({ + dbgs() << "Compact unwind: preparing " << G.getName() << " for prune\n"; + }); + + Section *EHFrameSec = G.findSectionByName(EHFrameSectionName); + + if (auto Err = splitCompactUnwindBlocks(G, *CUSec, CURecTraits::Size)) + return Err; + + LLVM_DEBUG({ + dbgs() << " Preparing " << CUSec->blocks_size() << " blocks in " + << CompactUnwindSectionName << "\n"; + }); + + for (auto *B : CUSec->blocks()) { + + // Find target function edge. + Edge *PCBeginEdge = nullptr; + for (auto &E : B->edges_at(CURecTraits::FnFieldOffset)) { + PCBeginEdge = &E; + break; + } + + if (!PCBeginEdge) + return make_error<JITLinkError>( + "In " + G.getName() + ", compact unwind record at " + + formatv("{0:x}", B->getAddress()) + " has no pc-begin edge"); + + if (!PCBeginEdge->getTarget().isDefined()) + return make_error<JITLinkError>( + "In " + G.getName() + ", compact unwind record at " + + formatv("{0:x}", B->getAddress()) + " points at external symbol " + + *PCBeginEdge->getTarget().getName()); + + auto &Fn = PCBeginEdge->getTarget(); + + if (!Fn.isDefined()) { + LLVM_DEBUG({ + dbgs() << "In " << CompactUnwindSectionName << " for " << G.getName() + << " encountered unexpected pc-edge to undefined symbol " + << Fn.getName() << "\n"; + }); + continue; + } else { + LLVM_DEBUG({ + dbgs() << " Found record for function "; + if (Fn.hasName()) + dbgs() << Fn.getName(); + else + dbgs() << "<anon @ " << Fn.getAddress() << '>'; + dbgs() << '\n'; + }); + } + + bool NeedsDWARF = CURecTraits::encodingSpecifiesDWARF( + CURecTraits::readEncoding(B->getContent())); + + auto &CURecSym = + G.addAnonymousSymbol(*B, 0, CURecTraits::Size, false, false); + + bool KeepAliveAlreadyPresent = false; + if (EHFrameSec) { + Edge *KeepAliveEdge = nullptr; + for (auto &E : Fn.getBlock().edges_at(0)) { + if (E.getKind() == Edge::KeepAlive && E.getTarget().isDefined() && + &E.getTarget().getBlock().getSection() == EHFrameSec) { + KeepAliveEdge = &E; + break; + } + } + + if (KeepAliveEdge) { + // Found a keep-alive edge to an FDE in the eh-frame. Switch the keep + // alive edge to point to the CU and if the CU needs DWARF then add + // an extra keep-alive edge from the CU to the FDE. + auto &FDE = KeepAliveEdge->getTarget(); + KeepAliveEdge->setTarget(CURecSym); + KeepAliveAlreadyPresent = true; + if (NeedsDWARF) { + LLVM_DEBUG({ + dbgs() << " Needs DWARF: adding keep-alive edge to FDE at " + << FDE.getAddress() << "\n"; + }); + B->addEdge(Edge::KeepAlive, 0, FDE, 0); + } + } else { + if (NeedsDWARF) + return make_error<JITLinkError>( + "In " + G.getName() + ", compact unwind recard ot " + + formatv("{0:x}", B->getAddress()) + + " needs DWARF, but no FDE was found"); + } + } else { + if (NeedsDWARF) + return make_error<JITLinkError>( + "In " + G.getName() + ", compact unwind recard ot " + + formatv("{0:x}", B->getAddress()) + " needs DWARF, but no " + + EHFrameSectionName + " section exists"); + } + + if (!KeepAliveAlreadyPresent) { + // No FDE edge. We'll need to add a new edge from the function back + // to the CU record. + Fn.getBlock().addEdge(Edge::KeepAlive, 0, CURecSym, 0); + } + } + + return Error::success(); + } + + /// Process all __compact_unwind records and reserve space for __unwind_info. + Error processAndReserveUnwindInfo(LinkGraph &G) { + // Bail out early if no unwind info. + Section *CUSec = G.findSectionByName(CompactUnwindSectionName); + if (!CUSec) + return Error::success(); + + // The __LD/__compact_unwind section is only used as input for the linker. + // We'll create a new __TEXT,__unwind_info section for unwind info output. + CUSec->setMemLifetime(orc::MemLifetime::NoAlloc); + + // Find / make a mach-header to act as the base for unwind-info offsets + // (and to report the arch / subarch to libunwind). + if (auto Err = getOrCreateCompactUnwindBase(G)) + return Err; + + // Error out if there's already unwind-info in the graph: We have no idea + // how to merge unwind-info sections. + if (G.findSectionByName(UnwindInfoSectionName)) + return make_error<JITLinkError>("In " + G.getName() + ", " + + UnwindInfoSectionName + + " already exists"); + + // Process the __compact_unwind section to build the Records vector that + // we'll use for writing the __unwind_info section. + if (auto Err = processCompactUnwind(G, *CUSec)) + return Err; + + // Calculate the size of __unwind_info. + size_t UnwindInfoSectionSize = + UnwindInfoSectionHeaderSize + + Personalities.size() * PersonalityEntrySize + + (NumSecondLevelPages + 1) * IndexEntrySize + NumLSDAs * LSDAEntrySize + + NumSecondLevelPages * SecondLevelPageHeaderSize + + Records.size() * SecondLevelPageEntrySize; + + LLVM_DEBUG({ + dbgs() << "In " << G.getName() << ", reserving " + << formatv("{0:x}", UnwindInfoSectionSize) << " bytes for " + << UnwindInfoSectionName << "\n"; + }); + + // Create the __unwind_info section and reserve space for it. + Section &UnwindInfoSec = + G.createSection(UnwindInfoSectionName, orc::MemProt::Read); + + auto UnwindInfoSectionContent = G.allocateBuffer(UnwindInfoSectionSize); + memset(UnwindInfoSectionContent.data(), 0, UnwindInfoSectionContent.size()); + auto &B = G.createMutableContentBlock( + UnwindInfoSec, UnwindInfoSectionContent, orc::ExecutorAddr(), 8, 0); + + // Add Keep-alive edges from the __unwind_info block to all of the target + // functions. + for (auto &R : Records) + B.addEdge(Edge::KeepAlive, 0, *R.Fn, 0); + + return Error::success(); + } + + Error writeUnwindInfo(LinkGraph &G) { + Section *CUSec = G.findSectionByName(CompactUnwindSectionName); + if (!CUSec || CUSec->empty()) + return Error::success(); + + Section *UnwindInfoSec = G.findSectionByName(UnwindInfoSectionName); + if (!UnwindInfoSec) + return make_error<JITLinkError>("In " + G.getName() + ", " + + UnwindInfoSectionName + + " missing after allocation"); + + if (UnwindInfoSec->blocks_size() != 1) + return make_error<JITLinkError>( + "In " + G.getName() + ", " + UnwindInfoSectionName + + " contains more than one block post-allocation"); + + LLVM_DEBUG( + { dbgs() << "Writing unwind info for " << G.getName() << "...\n"; }); + + mergeRecords(); + + auto &UnwindInfoBlock = **UnwindInfoSec->blocks().begin(); + auto Content = UnwindInfoBlock.getMutableContent(G); + BinaryStreamWriter Writer( + {reinterpret_cast<uint8_t *>(Content.data()), Content.size()}, + CURecTraits::Endianness); + + // __unwind_info format, from mach-o/compact_unwind_encoding.h on Darwin: + // + // #define UNWIND_SECTION_VERSION 1 + // struct unwind_info_section_header + // { + // uint32_t version; // UNWIND_SECTION_VERSION + // uint32_t commonEncodingsArraySectionOffset; + // uint32_t commonEncodingsArrayCount; + // uint32_t personalityArraySectionOffset; + // uint32_t personalityArrayCount; + // uint32_t indexSectionOffset; + // uint32_t indexCount; + // // compact_unwind_encoding_t[] + // // uint32_t personalities[] + // // unwind_info_section_header_index_entry[] + // // unwind_info_section_header_lsda_index_entry[] + // }; + + if (auto Err = writeHeader(G, Writer)) + return Err; + + // Skip common encodings: JITLink doesn't use them. + + if (auto Err = writePersonalities(G, Writer)) + return Err; + + // Calculate the offset to the LSDAs. + size_t SectionOffsetToLSDAs = + Writer.getOffset() + (NumSecondLevelPages + 1) * IndexEntrySize; + + // Calculate offset to the 1st second-level page. + size_t SectionOffsetToSecondLevelPages = + SectionOffsetToLSDAs + NumLSDAs * LSDAEntrySize; + + if (auto Err = writeIndexes(G, Writer, SectionOffsetToLSDAs, + SectionOffsetToSecondLevelPages)) + return Err; + + if (auto Err = writeLSDAs(G, Writer)) + return Err; + + if (auto Err = writeSecondLevelPages(G, Writer)) + return Err; + + LLVM_DEBUG({ + dbgs() << " Wrote " << formatv("{0:x}", Writer.getOffset()) + << " bytes of unwind info.\n"; + }); + + return Error::success(); + } + +private: + // Calculate the size of unwind-info. + static constexpr size_t MaxPersonalities = 4; + static constexpr size_t PersonalityShift = 28; + + static constexpr size_t UnwindInfoSectionHeaderSize = 4 * 7; + static constexpr size_t PersonalityEntrySize = 4; + static constexpr size_t IndexEntrySize = 3 * 4; + static constexpr size_t LSDAEntrySize = 2 * 4; + static constexpr size_t SecondLevelPageSize = 4096; + static constexpr size_t SecondLevelPageHeaderSize = 8; + static constexpr size_t SecondLevelPageEntrySize = 8; + static constexpr size_t NumRecordsPerSecondLevelPage = + (SecondLevelPageSize - SecondLevelPageHeaderSize) / + SecondLevelPageEntrySize; + + struct CompactUnwindRecord { + Symbol *Fn = nullptr; + uint32_t Size = 0; + uint32_t Encoding = 0; + Symbol *LSDA = nullptr; + Symbol *FDE = nullptr; + }; + + Error processCompactUnwind(LinkGraph &G, Section &CUSec) { + // TODO: Reset NumLSDAs, Personalities and CompactUnwindRecords if + // processing more than once. + assert(NumLSDAs == 0 && "NumLSDAs should be zero"); + assert(Records.empty() && "CompactUnwindRecords vector should be empty."); + assert(Personalities.empty() && "Personalities vector should be empty."); + + SmallVector<CompactUnwindRecord> NonUniquedRecords; + NonUniquedRecords.reserve(CUSec.blocks_size()); + + // Process __compact_unwind blocks. + for (auto *B : CUSec.blocks()) { + CompactUnwindRecord R; + R.Encoding = CURecTraits::readEncoding(B->getContent()); + for (auto &E : B->edges()) { + switch (E.getOffset()) { + case CURecTraits::FnFieldOffset: + // This could be the function-pointer, or the FDE keep-alive. Check + // the type to decide. + if (E.getKind() == Edge::KeepAlive) + R.FDE = &E.getTarget(); + else + R.Fn = &E.getTarget(); + break; + case CURecTraits::PersonalityFieldOffset: { + // Add the Personality to the Personalities map and update the + // encoding. + size_t PersonalityIdx = 0; + for (; PersonalityIdx != Personalities.size(); ++PersonalityIdx) + if (Personalities[PersonalityIdx] == &E.getTarget()) + break; + if (PersonalityIdx == MaxPersonalities) + return make_error<JITLinkError>( + "In " + G.getName() + + ", __compact_unwind contains too many personalities (max " + + formatv("{}", MaxPersonalities) + ")"); + if (PersonalityIdx == Personalities.size()) + Personalities.push_back(&E.getTarget()); + + R.Encoding |= (PersonalityIdx + 1) << PersonalityShift; + break; + } + case CURecTraits::LSDAFieldOffset: + ++NumLSDAs; + R.LSDA = &E.getTarget(); + break; + default: + return make_error<JITLinkError>("In " + G.getName() + + ", compact unwind record at " + + formatv("{0:x}", B->getAddress()) + + " has unrecognized edge at offset " + + formatv("{0:x}", E.getOffset())); + } + } + Records.push_back(R); + } + + // Sort the records into ascending order. + llvm::sort(Records, [](const CompactUnwindRecord &LHS, + const CompactUnwindRecord &RHS) { + return LHS.Fn->getAddress() < RHS.Fn->getAddress(); + }); + + // Calculate the number of second-level pages required. + NumSecondLevelPages = (Records.size() + NumRecordsPerSecondLevelPage - 1) / + NumRecordsPerSecondLevelPage; + + // Convert personality symbols to GOT entry pointers. + typename CURecTraits::GOTManager GOT(G); + for (auto &Personality : Personalities) + Personality = &GOT.getEntryForTarget(G, *Personality); + + LLVM_DEBUG({ + dbgs() << " In " << G.getName() << ", " << CompactUnwindSectionName + << ": raw records = " << Records.size() + << ", personalities = " << Personalities.size() + << ", lsdas = " << NumLSDAs << "\n"; + }); + + return Error::success(); + } + + void mergeRecords() { + SmallVector<CompactUnwindRecord> NonUniqued = std::move(Records); + Records.reserve(NonUniqued.size()); + + Records.push_back(NonUniqued.front()); + for (size_t I = 1; I != NonUniqued.size(); ++I) { + auto &Next = NonUniqued[I]; + auto &Last = Records.back(); + + bool NextNeedsDWARF = CURecTraits::encodingSpecifiesDWARF(Next.Encoding); + bool CannotBeMerged = CURecTraits::encodingCannotBeMerged(Next.Encoding); + if (NextNeedsDWARF || (Next.Encoding != Last.Encoding) || + CannotBeMerged || Next.LSDA || Last.LSDA) + Records.push_back(Next); + } + + // Recalculate derived values that may have changed. + NumSecondLevelPages = (Records.size() + NumRecordsPerSecondLevelPage - 1) / + NumRecordsPerSecondLevelPage; + } + + Error writeHeader(LinkGraph &G, BinaryStreamWriter &W) { + if (!isUInt<32>(NumSecondLevelPages + 1)) + return make_error<JITLinkError>("In " + G.getName() + ", too many " + + UnwindInfoSectionName + + "second-level pages required"); + + // Write __unwind_info header. + size_t IndexArrayOffset = UnwindInfoSectionHeaderSize + + Personalities.size() * PersonalityEntrySize; + + cantFail(W.writeInteger<uint32_t>(1)); + cantFail(W.writeInteger<uint32_t>(UnwindInfoSectionHeaderSize)); + cantFail(W.writeInteger<uint32_t>(0)); + cantFail(W.writeInteger<uint32_t>(UnwindInfoSectionHeaderSize)); + cantFail(W.writeInteger<uint32_t>(Personalities.size())); + cantFail(W.writeInteger<uint32_t>(IndexArrayOffset)); + cantFail(W.writeInteger<uint32_t>(NumSecondLevelPages + 1)); + + return Error::success(); + } + + Error writePersonalities(LinkGraph &G, BinaryStreamWriter &W) { + // Write personalities. + for (auto *PSym : Personalities) { + auto Delta = PSym->getAddress() - CompactUnwindBase->getAddress(); + if (!isUInt<32>(Delta)) + return makePersonalityRangeError(G, *PSym); + cantFail(W.writeInteger<uint32_t>(Delta)); + } + return Error::success(); + } + + Error writeIndexes(LinkGraph &G, BinaryStreamWriter &W, + size_t SectionOffsetToLSDAs, + size_t SectionOffsetToSecondLevelPages) { + // Assume that function deltas are ok in this method -- we'll error + // check all of them when we write the second level pages. + + // Write the header index entries. + size_t RecordIdx = 0; + size_t NumPreviousLSDAs = 0; + for (auto &R : Records) { + // If this record marks the start of a new second level page. + if (RecordIdx % NumRecordsPerSecondLevelPage == 0) { + auto FnDelta = R.Fn->getAddress() - CompactUnwindBase->getAddress(); + auto SecondLevelPageOffset = SectionOffsetToSecondLevelPages + + (RecordIdx / NumRecordsPerSecondLevelPage); + auto LSDAOffset = + SectionOffsetToLSDAs + NumPreviousLSDAs * LSDAEntrySize; + + cantFail(W.writeInteger<uint32_t>(FnDelta)); + cantFail(W.writeInteger<uint32_t>(SecondLevelPageOffset)); + cantFail(W.writeInteger<uint32_t>(LSDAOffset)); + } + if (R.LSDA) + ++NumPreviousLSDAs; + ++RecordIdx; + } + + // Write the index array terminator. + { + auto FnEndDelta = + Records.back().Fn->getRange().End - CompactUnwindBase->getAddress(); + + if (LLVM_UNLIKELY(!isUInt<32>(FnEndDelta))) + return make_error<JITLinkError>( + "In " + G.getName() + " " + UnwindInfoSectionName + + ", delta to end of functions " + + formatv("{0:x}", Records.back().Fn->getRange().End) + + " exceeds 32 bits"); + + cantFail(W.writeInteger<uint32_t>(FnEndDelta)); + cantFail(W.writeInteger<uint32_t>(0)); + cantFail(W.writeInteger<uint32_t>(SectionOffsetToSecondLevelPages)); + } + + return Error::success(); + } + + Error writeLSDAs(LinkGraph &G, BinaryStreamWriter &W) { + // As with writeIndexes, assume that function deltas are ok for now. + for (auto &R : Records) { + if (R.LSDA) { + auto FnDelta = R.Fn->getAddress() - CompactUnwindBase->getAddress(); + auto LSDADelta = R.LSDA->getAddress() - CompactUnwindBase->getAddress(); + + if (LLVM_UNLIKELY(!isUInt<32>(LSDADelta))) + return make_error<JITLinkError>( + "In " + G.getName() + " " + UnwindInfoSectionName + + ", delta to lsda at " + formatv("{0:x}", R.LSDA->getAddress()) + + " exceeds 32 bits"); + + cantFail(W.writeInteger<uint32_t>(FnDelta)); + cantFail(W.writeInteger<uint32_t>(LSDADelta)); + } + } + + return Error::success(); + } + + Error writeSecondLevelPages(LinkGraph &G, BinaryStreamWriter &W) { + size_t RecordIdx = 0; + + for (auto &R : Records) { + // When starting a new second-level page, write the page header: + // + // 2 : uint32_t -- UNWIND_SECOND_LEVEL_REGULAR + // 8 : uint16_t -- size of second level page table header + // count : uint16_t -- num entries in this second-level page + if (RecordIdx % NumRecordsPerSecondLevelPage == 0) { + constexpr uint32_t SecondLevelPageHeaderKind = 2; + constexpr uint16_t SecondLevelPageHeaderSize = 8; + uint16_t SecondLevelPageNumEntries = + std::min(Records.size() - RecordIdx, NumRecordsPerSecondLevelPage); + + cantFail(W.writeInteger<uint32_t>(SecondLevelPageHeaderKind)); + cantFail(W.writeInteger<uint16_t>(SecondLevelPageHeaderSize)); + cantFail(W.writeInteger<uint16_t>(SecondLevelPageNumEntries)); + } + + // Write entry. + auto FnDelta = R.Fn->getAddress() - CompactUnwindBase->getAddress(); + + if (LLVM_UNLIKELY(!isUInt<32>(FnDelta))) + return make_error<JITLinkError>( + "In " + G.getName() + " " + UnwindInfoSectionName + + ", delta to function at " + formatv("{0:x}", R.Fn->getAddress()) + + " exceeds 32 bits"); + + cantFail(W.writeInteger<uint32_t>(FnDelta)); + cantFail(W.writeInteger<uint32_t>(R.Encoding)); + + ++RecordIdx; + } + + return Error::success(); + } + + Error getOrCreateCompactUnwindBase(LinkGraph &G) { + auto Name = G.intern("__jitlink$libunwind_dso_base"); + CompactUnwindBase = G.findAbsoluteSymbolByName(Name); + if (!CompactUnwindBase) { + if (auto LocalCUBase = getOrCreateLocalMachOHeader(G)) { + CompactUnwindBase = &*LocalCUBase; + auto &B = LocalCUBase->getBlock(); + G.addDefinedSymbol(B, 0, *Name, B.getSize(), Linkage::Strong, + Scope::Local, false, true); + } else + return LocalCUBase.takeError(); + } + CompactUnwindBase->setLive(true); + return Error::success(); + } + + Error makePersonalityRangeError(LinkGraph &G, Symbol &PSym) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "In " << G.getName() << " " << UnwindInfoSectionName + << ", personality "; + if (PSym.hasName()) + ErrStream << PSym.getName() << " "; + ErrStream << "at " << PSym.getAddress() + << " is out of 32-bit delta range of compact-unwind base at " + << CompactUnwindBase->getAddress(); + } + return make_error<JITLinkError>(std::move(ErrMsg)); + } + + StringRef CompactUnwindSectionName; + StringRef UnwindInfoSectionName; + StringRef EHFrameSectionName; + Symbol *CompactUnwindBase = nullptr; + + size_t NumLSDAs = 0; + size_t NumSecondLevelPages = 0; + SmallVector<Symbol *, MaxPersonalities> Personalities; + SmallVector<CompactUnwindRecord> Records; +}; + +} // end namespace jitlink +} // end namespace llvm + +#undef DEBUG_TYPE + +#endif // LIB_EXECUTIONENGINE_JITLINK_COMPACTUNWINDSUPPORTIMPL_H diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp index 3e757f7..179e458c 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -733,121 +733,5 @@ Error MachOLinkGraphBuilder::graphifyCStringSection( return Error::success(); } -Error CompactUnwindSplitter::operator()(LinkGraph &G) { - auto *CUSec = G.findSectionByName(CompactUnwindSectionName); - if (!CUSec) - return Error::success(); - - if (!G.getTargetTriple().isOSBinFormatMachO()) - return make_error<JITLinkError>( - "Error linking " + G.getName() + - ": compact unwind splitting not supported on non-macho target " + - G.getTargetTriple().str()); - - unsigned CURecordSize = 0; - unsigned PersonalityEdgeOffset = 0; - unsigned LSDAEdgeOffset = 0; - switch (G.getTargetTriple().getArch()) { - case Triple::aarch64: - case Triple::x86_64: - // 64-bit compact-unwind record format: - // Range start: 8 bytes. - // Range size: 4 bytes. - // CU encoding: 4 bytes. - // Personality: 8 bytes. - // LSDA: 8 bytes. - CURecordSize = 32; - PersonalityEdgeOffset = 16; - LSDAEdgeOffset = 24; - break; - default: - return make_error<JITLinkError>( - "Error linking " + G.getName() + - ": compact unwind splitting not supported on " + - G.getTargetTriple().getArchName()); - } - - std::vector<Block *> OriginalBlocks(CUSec->blocks().begin(), - CUSec->blocks().end()); - LLVM_DEBUG({ - dbgs() << "In " << G.getName() << " splitting compact unwind section " - << CompactUnwindSectionName << " containing " - << OriginalBlocks.size() << " initial blocks...\n"; - }); - - while (!OriginalBlocks.empty()) { - auto *B = OriginalBlocks.back(); - OriginalBlocks.pop_back(); - - if (B->getSize() == 0) { - LLVM_DEBUG({ - dbgs() << " Skipping empty block at " - << formatv("{0:x16}", B->getAddress()) << "\n"; - }); - continue; - } - - unsigned NumBlocks = B->getSize() / CURecordSize; - - LLVM_DEBUG({ - dbgs() << " Splitting block at " << formatv("{0:x16}", B->getAddress()) - << " into " << NumBlocks << " compact unwind record(s)\n"; - }); - - if (B->getSize() % CURecordSize) - return make_error<JITLinkError>( - "Error splitting compact unwind record in " + G.getName() + - ": block at " + formatv("{0:x}", B->getAddress()) + " has size " + - formatv("{0:x}", B->getSize()) + - " (not a multiple of CU record size of " + - formatv("{0:x}", CURecordSize) + ")"); - - auto Blocks = - G.splitBlock(*B, map_range(seq(1U, NumBlocks), [=](Edge::OffsetT Idx) { - return Idx * CURecordSize; - })); - - for (auto *CURec : Blocks) { - bool AddedKeepAlive = false; - - for (auto &E : CURec->edges()) { - if (E.getOffset() == 0) { - LLVM_DEBUG({ - dbgs() << " Updating compact unwind record at " - << CURec->getAddress() << " to point to " - << (E.getTarget().hasName() ? *E.getTarget().getName() - : StringRef()) - << " (at " << E.getTarget().getAddress() << ")\n"; - }); - - if (E.getTarget().isExternal()) - return make_error<JITLinkError>( - "Error adding keep-alive edge for compact unwind record at " + - formatv("{0:x}", CURec->getAddress()) + ": target " + - *E.getTarget().getName() + " is an external symbol"); - auto &TgtBlock = E.getTarget().getBlock(); - auto &CURecSym = - G.addAnonymousSymbol(*CURec, 0, CURecordSize, false, false); - TgtBlock.addEdge(Edge::KeepAlive, 0, CURecSym, 0); - AddedKeepAlive = true; - } else if (E.getOffset() != PersonalityEdgeOffset && - E.getOffset() != LSDAEdgeOffset) - return make_error<JITLinkError>( - "Unexpected edge at offset " + formatv("{0:x}", E.getOffset()) + - " in compact unwind record at " + - formatv("{0:x}", CURec->getAddress())); - } - - if (!AddedKeepAlive) - return make_error<JITLinkError>( - "Error adding keep-alive edge for compact unwind record at " + - formatv("{0:x}", CURec->getAddress()) + - ": no outgoing target edge at offset 0"); - } - } - - return Error::success(); -} - } // end namespace jitlink } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h index 6afa012..343218e 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -236,17 +236,6 @@ private: StringMap<SectionParserFunction> CustomSectionParserFunctions; }; -/// A pass to split up __LD,__compact_unwind sections. -class CompactUnwindSplitter { -public: - CompactUnwindSplitter(StringRef CompactUnwindSectionName) - : CompactUnwindSectionName(CompactUnwindSectionName) {} - Error operator()(LinkGraph &G); - -private: - StringRef CompactUnwindSectionName; -}; - } // end namespace jitlink } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 29061ff..f9f2f4e 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -14,6 +14,7 @@ #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" #include "llvm/ExecutionEngine/JITLink/aarch64.h" +#include "CompactUnwindSupport.h" #include "DefineExternalSectionStartAndEndSymbols.h" #include "MachOLinkGraphBuilder.h" @@ -625,6 +626,27 @@ static Error applyPACSigningToModInitPointers(LinkGraph &G) { return Error::success(); } +struct CompactUnwindTraits_MachO_arm64 + : public CompactUnwindTraits<CompactUnwindTraits_MachO_arm64, + /* PointerSize = */ 8> { + // FIXME: Reinstate once we no longer need the MSVC workaround. See + // FIXME for CompactUnwindTraits in CompactUnwindSupport.h. + // constexpr static size_t PointerSize = 8; + + constexpr static endianness Endianness = endianness::little; + + constexpr static uint32_t EncodingModeMask = 0x0f000000; + + using GOTManager = aarch64::GOTTableManager; + + static bool encodingSpecifiesDWARF(uint32_t Encoding) { + constexpr uint32_t DWARFMode = 0x03000000; + return (Encoding & EncodingModeMask) == DWARFMode; + } + + static bool encodingCannotBeMerged(uint32_t Encoding) { return false; } +}; + void link_MachO_arm64(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) { @@ -637,16 +659,21 @@ void link_MachO_arm64(std::unique_ptr<LinkGraph> G, else Config.PrePrunePasses.push_back(markAllSymbolsLive); - // Add compact unwind splitter pass. - Config.PrePrunePasses.push_back( - CompactUnwindSplitter("__LD,__compact_unwind")); - // Add eh-frame passes. - // FIXME: Prune eh-frames for which compact-unwind is available once - // we support compact-unwind registration with libunwind. Config.PrePrunePasses.push_back(createEHFrameSplitterPass_MachO_arm64()); Config.PrePrunePasses.push_back(createEHFrameEdgeFixerPass_MachO_arm64()); + // Create a compact-unwind manager for use in passes below. + auto CompactUnwindMgr = + std::make_shared<CompactUnwindManager<CompactUnwindTraits_MachO_arm64>>( + "__LD,__compact_unwind", "__TEXT,__unwind_info", + "__TEXT,__eh_frame"); + + // Add compact unwind prepare pass. + Config.PrePrunePasses.push_back([CompactUnwindMgr](LinkGraph &G) { + return CompactUnwindMgr->prepareForPrune(G); + }); + // Resolve any external section start / end symbols. Config.PostAllocationPasses.push_back( createDefineExternalSectionStartAndEndSymbolsPass( @@ -663,6 +690,16 @@ void link_MachO_arm64(std::unique_ptr<LinkGraph> G, Config.PreFixupPasses.push_back( aarch64::lowerPointer64AuthEdgesToSigningFunction); } + + // Reserve unwind-info space. + Config.PostPrunePasses.push_back([CompactUnwindMgr](LinkGraph &G) { + return CompactUnwindMgr->processAndReserveUnwindInfo(G); + }); + + // Translate compact-unwind to unwind-info. + Config.PreFixupPasses.push_back([CompactUnwindMgr](LinkGraph &G) { + return CompactUnwindMgr->writeUnwindInfo(G); + }); } if (auto Err = Ctx->modifyPassConfig(*G, Config)) diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index 9547266..218f8ce 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -14,6 +14,7 @@ #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" +#include "CompactUnwindSupport.h" #include "DefineExternalSectionStartAndEndSymbols.h" #include "MachOLinkGraphBuilder.h" @@ -500,26 +501,56 @@ Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromMachOObject_x86_64( .buildGraph(); } +struct CompactUnwindTraits_MachO_x86_64 + : public CompactUnwindTraits<CompactUnwindTraits_MachO_x86_64, + /* PointerSize = */ 8> { + // FIXME: Reinstate once we no longer need the MSVC workaround. See + // FIXME for CompactUnwindTraits in CompactUnwindSupport.h. + // constexpr static size_t PointerSize = 8; + + constexpr static endianness Endianness = endianness::little; + + constexpr static uint32_t EncodingModeMask = 0x0f000000; + + using GOTManager = x86_64::GOTTableManager; + + static bool encodingSpecifiesDWARF(uint32_t Encoding) { + constexpr uint32_t DWARFMode = 0x04000000; + return (Encoding & EncodingModeMask) == DWARFMode; + } + + static bool encodingCannotBeMerged(uint32_t Encoding) { + constexpr uint32_t StackIndirectMode = 0x03000000; + return (Encoding & EncodingModeMask) == StackIndirectMode; + } +}; + void link_MachO_x86_64(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) { PassConfiguration Config; if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { - // Add eh-frame passes. - Config.PrePrunePasses.push_back(createEHFrameSplitterPass_MachO_x86_64()); - Config.PrePrunePasses.push_back(createEHFrameEdgeFixerPass_MachO_x86_64()); - - // Add compact unwind splitter pass. - Config.PrePrunePasses.push_back( - CompactUnwindSplitter("__LD,__compact_unwind")); - // Add a mark-live pass. if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) Config.PrePrunePasses.push_back(std::move(MarkLive)); else Config.PrePrunePasses.push_back(markAllSymbolsLive); + // Add eh-frame passes. + Config.PrePrunePasses.push_back(createEHFrameSplitterPass_MachO_x86_64()); + Config.PrePrunePasses.push_back(createEHFrameEdgeFixerPass_MachO_x86_64()); + + // Create a compact-unwind manager for use in passes below. + auto CompactUnwindMgr = std::make_shared< + CompactUnwindManager<CompactUnwindTraits_MachO_x86_64>>( + "__LD,__compact_unwind", "__TEXT,__unwind_info", "__TEXT,__eh_frame"); + + // Add compact unwind prepare pass. + Config.PrePrunePasses.push_back([CompactUnwindMgr](LinkGraph &G) { + return CompactUnwindMgr->prepareForPrune(G); + }); + // Resolve any external section start / end symbols. Config.PostAllocationPasses.push_back( createDefineExternalSectionStartAndEndSymbolsPass( @@ -528,6 +559,16 @@ void link_MachO_x86_64(std::unique_ptr<LinkGraph> G, // Add an in-place GOT/Stubs pass. Config.PostPrunePasses.push_back(buildGOTAndStubs_MachO_x86_64); + // Reserve space for unwind-info. + Config.PostPrunePasses.push_back([CompactUnwindMgr](LinkGraph &G) { + return CompactUnwindMgr->processAndReserveUnwindInfo(G); + }); + + // Translate compact-unwind to unwind-info. + Config.PreFixupPasses.push_back([CompactUnwindMgr](LinkGraph &G) { + return CompactUnwindMgr->writeUnwindInfo(G); + }); + // Add GOT/Stubs optimizer pass. Config.PreFixupPasses.push_back(x86_64::optimizeGOTAndStubAccesses); } diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt index 2ab5d6d..8a86629 100644 --- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -57,6 +57,7 @@ add_llvm_component_library(LLVMOrcJIT ExecutorProcessControl.cpp TaskDispatch.cpp ThreadSafeModule.cpp + UnwindInfoRegistrationPlugin.cpp RedirectionManager.cpp JITLinkRedirectableSymbolManager.cpp ReOptimizeLayer.cpp diff --git a/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp b/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp index 5d2f3cd..c4d65af 100644 --- a/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp @@ -33,6 +33,9 @@ irManglingOptionsFromTargetOptions(const TargetOptions &Opts) { /// Compile a Module to an ObjectFile. Expected<SimpleCompiler::CompileResult> SimpleCompiler::operator()(Module &M) { + if (M.getDataLayout().isDefault()) + M.setDataLayout(TM.createDataLayout()); + CompileResult CachedObject = tryToLoadFromObjectCache(M); if (CachedObject) return std::move(CachedObject); diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp index aa79968..9526845 100644 --- a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp @@ -45,6 +45,7 @@ SelfExecutorProcessControl::SelfExecutorProcessControl( this->DylibMgr = this; this->JDI = {ExecutorAddr::fromPtr(jitDispatchViaWrapperFunctionManager), ExecutorAddr::fromPtr(this)}; + this->UnwindInfoMgr = UnwindInfoManager::TryCreate(); if (this->TargetTriple.isOSBinFormatMachO()) GlobalManglingPrefix = '_'; @@ -52,6 +53,8 @@ SelfExecutorProcessControl::SelfExecutorProcessControl( ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionWrapper); this->BootstrapSymbols[rt::DeregisterEHFrameSectionWrapperName] = ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionWrapper); + if (this->UnwindInfoMgr) + this->UnwindInfoMgr->addBootstrapSymbols(this->BootstrapSymbols); } Expected<std::unique_ptr<SelfExecutorProcessControl>> diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index b1dadba..ab7f854 100644 --- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -21,6 +21,7 @@ #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" +#include "llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IRBuilder.h" @@ -833,14 +834,7 @@ Error LLJITBuilderState::prepareForConstruction() { CreateObjectLinkingLayer = [](ExecutionSession &ES, const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> { - auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES); - if (auto EHFrameRegistrar = EPCEHFrameRegistrar::Create(ES)) - ObjLinkingLayer->addPlugin( - std::make_unique<EHFrameRegistrationPlugin>( - ES, std::move(*EHFrameRegistrar))); - else - return EHFrameRegistrar.takeError(); - return std::move(ObjLinkingLayer); + return std::make_unique<ObjectLinkingLayer>(ES); }; } } @@ -1225,6 +1219,32 @@ Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J) { auto &PlatformJD = J.getExecutionSession().createBareJITDylib("<Platform>"); PlatformJD.addToLinkOrder(*ProcessSymbolsJD); + if (auto *OLL = dyn_cast<ObjectLinkingLayer>(&J.getObjLinkingLayer())) { + + bool CompactUnwindInfoSupported = false; + + // Enable compact-unwind support if possible. + if (J.getTargetTriple().isOSDarwin() || + J.getTargetTriple().isOSBinFormatMachO()) { + if (auto UIRP = UnwindInfoRegistrationPlugin::Create( + J.getIRCompileLayer(), PlatformJD)) { + CompactUnwindInfoSupported = true; + OLL->addPlugin(std::move(*UIRP)); + } else + consumeError(UIRP.takeError()); + } + + // Otherwise fall back to standard unwind registration. + if (!CompactUnwindInfoSupported) { + auto &ES = J.getExecutionSession(); + if (auto EHFrameRegistrar = EPCEHFrameRegistrar::Create(ES)) + OLL->addPlugin(std::make_unique<EHFrameRegistrationPlugin>( + ES, std::move(*EHFrameRegistrar))); + else + return EHFrameRegistrar.takeError(); + } + } + J.setPlatformSupport( std::make_unique<GenericLLVMIRPlatformSupport>(J, PlatformJD)); diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp index 54a25c0..fef3ff9 100644 --- a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp @@ -64,5 +64,19 @@ const char *RunAsIntFunctionWrapperName = "__llvm_orc_bootstrap_run_as_int_function_wrapper"; } // end namespace rt +namespace rt_alt { +const char *UnwindInfoManagerInstanceName = + "orc_rt_alt_UnwindInfoManager_Instance"; +const char *UnwindInfoManagerFindSectionsHelperName = + "orc_rt_alt_UnwindInfoManager_findSectionsHelper"; +const char *UnwindInfoManagerEnableWrapperName = + "orc_rt_alt_UnwindInfoManager_enable"; +const char *UnwindInfoManagerDisableWrapperName = + "orc_rt_alt_UnwindInfoManager_disable"; +const char *UnwindInfoManagerRegisterActionName = + "orc_rt_alt_UnwindInfoManager_register"; +const char *UnwindInfoManagerDeregisterActionName = + "orc_rt_alt_UnwindInfoManager_deregister"; +} // end namespace rt_alt } // end namespace orc } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt index 3d1dfe7..ffc1bbf 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt @@ -20,6 +20,7 @@ add_llvm_component_library(LLVMOrcTargetProcess SimpleExecutorMemoryManager.cpp SimpleRemoteEPCServer.cpp TargetExecutionUtils.cpp + UnwindInfoManager.cpp ADDITIONAL_HEADER_DIRS ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.cpp new file mode 100644 index 0000000..28cbd45 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.cpp @@ -0,0 +1,185 @@ +//===------- UnwindInfoManager.cpp - Register unwind info sections --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include "llvm/Support/DynamicLibrary.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; + +static orc::shared::CWrapperFunctionResult +llvm_orc_rt_alt_UnwindInfoManager_enable(const char *Data, uint64_t Size) { + return WrapperFunction<SPSError(SPSExecutorAddr, SPSExecutorAddr)>::handle( + Data, Size, + [](ExecutorAddr Instance, ExecutorAddr FindFn) { + return Instance.toPtr<UnwindInfoManager *>()->enable( + FindFn.toPtr<void *>()); + }) + .release(); +} + +static orc::shared::CWrapperFunctionResult +llvm_orc_rt_alt_UnwindInfoManager_disable(const char *Data, uint64_t Size) { + return WrapperFunction<SPSError(SPSExecutorAddr)>::handle( + Data, Size, + [](ExecutorAddr Instance) { + return Instance.toPtr<UnwindInfoManager *>()->disable(); + }) + .release(); +} + +static orc::shared::CWrapperFunctionResult +llvm_orc_rt_alt_UnwindInfoManager_register(const char *Data, uint64_t Size) { + using SPSSig = + SPSError(SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>, + SPSExecutorAddr, SPSExecutorAddrRange, SPSExecutorAddrRange); + + return WrapperFunction<SPSSig>::handle( + Data, Size, + [](ExecutorAddr Instance, + std::vector<ExecutorAddrRange> CodeRanges, ExecutorAddr DSOBase, + ExecutorAddrRange DWARFRange, + ExecutorAddrRange CompactUnwindRange) { + return Instance.toPtr<UnwindInfoManager *>()->registerSections( + CodeRanges, DSOBase, DWARFRange, CompactUnwindRange); + }) + .release(); +} + +static orc::shared::CWrapperFunctionResult +llvm_orc_rt_alt_UnwindInfoManager_deregister(const char *Data, uint64_t Size) { + using SPSSig = SPSError(SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>); + + return WrapperFunction<SPSSig>::handle( + Data, Size, + [](ExecutorAddr Instance, + std::vector<ExecutorAddrRange> CodeRanges) { + return Instance.toPtr<UnwindInfoManager *>()->deregisterSections( + CodeRanges); + }) + .release(); +} + +namespace llvm::orc { + +const char *UnwindInfoManager::AddFnName = + "__unw_add_find_dynamic_unwind_sections"; +const char *UnwindInfoManager::RemoveFnName = + "__unw_remove_find_dynamic_unwind_sections"; + +std::unique_ptr<UnwindInfoManager> UnwindInfoManager::TryCreate() { + std::string ErrMsg; + auto DL = sys::DynamicLibrary::getPermanentLibrary(nullptr, &ErrMsg); + if (!DL.isValid()) + return nullptr; + + auto AddFindDynamicUnwindSections = + (int (*)(void *))DL.getAddressOfSymbol(AddFnName); + if (!AddFindDynamicUnwindSections) + return nullptr; + + auto RemoveFindDynamicUnwindSections = + (int (*)(void *))DL.getAddressOfSymbol(RemoveFnName); + if (!RemoveFindDynamicUnwindSections) + return nullptr; + + return std::unique_ptr<UnwindInfoManager>(new UnwindInfoManager( + AddFindDynamicUnwindSections, RemoveFindDynamicUnwindSections)); +} + +Error UnwindInfoManager::shutdown() { return Error::success(); } + +void UnwindInfoManager::addBootstrapSymbols(StringMap<ExecutorAddr> &M) { + M[rt_alt::UnwindInfoManagerInstanceName] = ExecutorAddr::fromPtr(this); + M[rt_alt::UnwindInfoManagerFindSectionsHelperName] = + ExecutorAddr::fromPtr(&findSectionsHelper); + M[rt_alt::UnwindInfoManagerEnableWrapperName] = + ExecutorAddr::fromPtr(llvm_orc_rt_alt_UnwindInfoManager_enable); + M[rt_alt::UnwindInfoManagerDisableWrapperName] = + ExecutorAddr::fromPtr(llvm_orc_rt_alt_UnwindInfoManager_disable); + M[rt_alt::UnwindInfoManagerRegisterActionName] = + ExecutorAddr::fromPtr(llvm_orc_rt_alt_UnwindInfoManager_register); + M[rt_alt::UnwindInfoManagerDeregisterActionName] = + ExecutorAddr::fromPtr(llvm_orc_rt_alt_UnwindInfoManager_deregister); +} + +Error UnwindInfoManager::enable(void *FindDynamicUnwindSections) { + LLVM_DEBUG(dbgs() << "Enabling UnwindInfoManager.\n"); + + if (auto Err = AddFindDynamicUnwindSections(FindDynamicUnwindSections)) + return make_error<StringError>(Twine("Could not register function via ") + + AddFnName + + ", error code = " + Twine(Err), + inconvertibleErrorCode()); + + this->FindDynamicUnwindSections = FindDynamicUnwindSections; + return Error::success(); +} + +Error UnwindInfoManager::disable(void) { + LLVM_DEBUG(dbgs() << "Disabling UnwindInfoManager.\n"); + + if (FindDynamicUnwindSections) + if (auto Err = RemoveFindDynamicUnwindSections(FindDynamicUnwindSections)) + return make_error<StringError>( + Twine("Could not deregister function via ") + RemoveFnName + + "error code = " + Twine(Err), + inconvertibleErrorCode()); + + FindDynamicUnwindSections = nullptr; + return Error::success(); +} + +Error UnwindInfoManager::registerSections( + ArrayRef<ExecutorAddrRange> CodeRanges, ExecutorAddr DSOBase, + ExecutorAddrRange DWARFEHFrame, ExecutorAddrRange CompactUnwind) { + std::lock_guard<std::mutex> Lock(M); + for (auto &R : CodeRanges) + UWSecs[R.Start.getValue()] = UnwindSections{ + DSOBase.getValue(), DWARFEHFrame.Start.getValue(), DWARFEHFrame.size(), + CompactUnwind.Start.getValue(), CompactUnwind.size()}; + return Error::success(); +} + +Error UnwindInfoManager::deregisterSections( + ArrayRef<ExecutorAddrRange> CodeRanges) { + std::lock_guard<std::mutex> Lock(M); + for (auto &R : CodeRanges) { + auto I = UWSecs.find(R.Start.getValue()); + if (I == UWSecs.end()) + return make_error<StringError>( + "No unwind-info sections registered for range " + + formatv("{0:x} - {1:x}", R.Start, R.End), + inconvertibleErrorCode()); + UWSecs.erase(I); + } + return Error::success(); +} + +int UnwindInfoManager::findSections(uintptr_t Addr, UnwindSections *Info) { + std::lock_guard<std::mutex> Lock(M); + auto I = UWSecs.upper_bound(Addr); + if (I == UWSecs.begin()) + return 0; + --I; + *Info = I->second; + return 1; +} + +int UnwindInfoManager::findSectionsHelper(UnwindInfoManager *Instance, + uintptr_t Addr, + UnwindSections *Info) { + return Instance->findSections(Addr, Info); +} + +} // namespace llvm::orc diff --git a/llvm/lib/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.cpp new file mode 100644 index 0000000..0073f3d --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.cpp @@ -0,0 +1,238 @@ +//===----- UnwindInfoRegistrationPlugin.cpp - libunwind registration ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h" + +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h" +#include "llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Module.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm::jitlink; + +static const char *FindDynamicUnwindSectionsFunctionName = + "_orc_rt_alt_find_dynamic_unwind_sections"; + +namespace llvm::orc { + +Expected<std::shared_ptr<UnwindInfoRegistrationPlugin>> +UnwindInfoRegistrationPlugin::Create(IRLayer &IRL, JITDylib &PlatformJD, + ExecutorAddr Instance, + ExecutorAddr FindHelper, + ExecutorAddr Enable, ExecutorAddr Disable, + ExecutorAddr Register, + ExecutorAddr Deregister) { + + auto &ES = IRL.getExecutionSession(); + + // Build bouncer module. + auto M = makeBouncerModule(ES); + if (!M) + return M.takeError(); + + auto BouncerRT = PlatformJD.createResourceTracker(); + auto RemoveBouncerModule = make_scope_exit([&]() { + if (auto Err = BouncerRT->remove()) + ES.reportError(std::move(Err)); + }); + + if (auto Err = PlatformJD.define(absoluteSymbols( + {{ES.intern(rt_alt::UnwindInfoManagerInstanceName), + ExecutorSymbolDef(Instance, JITSymbolFlags())}, + {ES.intern(rt_alt::UnwindInfoManagerFindSectionsHelperName), + ExecutorSymbolDef(FindHelper, JITSymbolFlags::Callable)}}))) + return std::move(Err); + + if (auto Err = IRL.add(BouncerRT, std::move(*M))) + return Err; + + auto FindUnwindSections = + ES.lookup({&PlatformJD}, FindDynamicUnwindSectionsFunctionName); + if (!FindUnwindSections) + return FindUnwindSections.takeError(); + + using namespace shared; + using SPSEnableSig = SPSError(SPSExecutorAddr, SPSExecutorAddr); + Error CallErr = Error::success(); + if (auto Err = ES.callSPSWrapper<SPSEnableSig>( + Enable, CallErr, Instance, FindUnwindSections->getAddress())) { + consumeError(std::move(CallErr)); + return std::move(Err); + } + + if (CallErr) + return std::move(CallErr); + + RemoveBouncerModule.release(); + + return std::shared_ptr<UnwindInfoRegistrationPlugin>( + new UnwindInfoRegistrationPlugin(ES, Instance, Disable, Register, + Deregister)); +} + +Expected<std::shared_ptr<UnwindInfoRegistrationPlugin>> +UnwindInfoRegistrationPlugin::Create(IRLayer &IRL, JITDylib &PlatformJD) { + + ExecutorAddr Instance, FindHelper, Enable, Disable, Register, Deregister; + + auto &EPC = IRL.getExecutionSession().getExecutorProcessControl(); + if (auto Err = EPC.getBootstrapSymbols( + {{Instance, rt_alt::UnwindInfoManagerInstanceName}, + {FindHelper, rt_alt::UnwindInfoManagerFindSectionsHelperName}, + {Enable, rt_alt::UnwindInfoManagerEnableWrapperName}, + {Disable, rt_alt::UnwindInfoManagerDisableWrapperName}, + {Register, rt_alt::UnwindInfoManagerRegisterActionName}, + {Deregister, rt_alt::UnwindInfoManagerDeregisterActionName}})) + return std::move(Err); + + return Create(IRL, PlatformJD, Instance, FindHelper, Enable, Disable, + Register, Deregister); +} + +UnwindInfoRegistrationPlugin::~UnwindInfoRegistrationPlugin() { + using namespace shared; + using SPSDisableSig = SPSError(SPSExecutorAddr); + Error CallErr = Error::success(); + if (auto Err = ES.callSPSWrapper<SPSDisableSig>(Disable, CallErr, Instance)) { + consumeError(std::move(CallErr)); + ES.reportError(std::move(Err)); + } + if (CallErr) + ES.reportError(std::move(CallErr)); +} + +void UnwindInfoRegistrationPlugin::modifyPassConfig( + MaterializationResponsibility &MR, LinkGraph &G, + PassConfiguration &PassConfig) { + + PassConfig.PostFixupPasses.push_back( + [this](LinkGraph &G) { return addUnwindInfoRegistrationActions(G); }); +} + +Expected<ThreadSafeModule> +UnwindInfoRegistrationPlugin::makeBouncerModule(ExecutionSession &ES) { + auto Ctx = std::make_unique<LLVMContext>(); + auto M = std::make_unique<Module>("__libunwind_find_unwind_bouncer", *Ctx); + M->setTargetTriple(ES.getTargetTriple().str()); + + auto EscapeName = [](const char *N) { return std::string("\01") + N; }; + + auto *PtrTy = PointerType::getUnqual(*Ctx); + auto *OpaqueStructTy = StructType::create(*Ctx, "UnwindInfoMgr"); + auto *UnwindMgrInstance = new GlobalVariable( + *M, OpaqueStructTy, true, GlobalValue::ExternalLinkage, nullptr, + EscapeName(rt_alt::UnwindInfoManagerInstanceName)); + + auto *Int64Ty = Type::getInt64Ty(*Ctx); + auto *FindHelperTy = FunctionType::get(Int64Ty, {PtrTy, PtrTy, PtrTy}, false); + auto *FindHelperFn = Function::Create( + FindHelperTy, GlobalValue::ExternalLinkage, + EscapeName(rt_alt::UnwindInfoManagerFindSectionsHelperName), *M); + + auto *FindFnTy = FunctionType::get(Int64Ty, {PtrTy, PtrTy}, false); + auto *FindFn = + Function::Create(FindFnTy, GlobalValue::ExternalLinkage, + EscapeName(FindDynamicUnwindSectionsFunctionName), *M); + auto *EntryBlock = BasicBlock::Create(M->getContext(), StringRef(), FindFn); + IRBuilder<> IB(EntryBlock); + + std::vector<Value *> FindHelperArgs; + FindHelperArgs.push_back(UnwindMgrInstance); + for (auto &Arg : FindFn->args()) + FindHelperArgs.push_back(&Arg); + + IB.CreateRet(IB.CreateCall(FindHelperFn, FindHelperArgs)); + + return ThreadSafeModule(std::move(M), std::move(Ctx)); +} + +Error UnwindInfoRegistrationPlugin::addUnwindInfoRegistrationActions( + LinkGraph &G) { + ExecutorAddrRange EHFrameRange, UnwindInfoRange; + + std::vector<Block *> CodeBlocks; + + auto ScanUnwindInfoSection = [&](Section &Sec, ExecutorAddrRange &SecRange) { + if (Sec.empty()) + return; + + SecRange.Start = (*Sec.blocks().begin())->getAddress(); + for (auto *B : Sec.blocks()) { + auto R = B->getRange(); + SecRange.Start = std::min(SecRange.Start, R.Start); + SecRange.End = std::max(SecRange.End, R.End); + for (auto &E : B->edges()) { + if (E.getKind() != Edge::KeepAlive || !E.getTarget().isDefined()) + continue; + auto &TargetBlock = E.getTarget().getBlock(); + auto &TargetSection = TargetBlock.getSection(); + if ((TargetSection.getMemProt() & MemProt::Exec) == MemProt::Exec) + CodeBlocks.push_back(&TargetBlock); + } + } + }; + + if (auto *EHFrame = G.findSectionByName(MachOEHFrameSectionName)) + ScanUnwindInfoSection(*EHFrame, EHFrameRange); + + if (auto *UnwindInfo = G.findSectionByName(MachOCompactUnwindInfoSectionName)) + ScanUnwindInfoSection(*UnwindInfo, UnwindInfoRange); + + if (CodeBlocks.empty()) + return Error::success(); + + if ((EHFrameRange == ExecutorAddrRange() && + UnwindInfoRange == ExecutorAddrRange())) + return Error::success(); + + llvm::sort(CodeBlocks, [](const Block *LHS, const Block *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + + SmallVector<ExecutorAddrRange> CodeRanges; + for (auto *B : CodeBlocks) { + if (CodeRanges.empty() || CodeRanges.back().End != B->getAddress()) + CodeRanges.push_back(B->getRange()); + else + CodeRanges.back().End = B->getRange().End; + } + + ExecutorAddr DSOBase; + if (auto *DSOBaseSym = G.findAbsoluteSymbolByName(DSOBaseName)) + DSOBase = DSOBaseSym->getAddress(); + else if (auto *DSOBaseSym = G.findExternalSymbolByName(DSOBaseName)) + DSOBase = DSOBaseSym->getAddress(); + else if (auto *DSOBaseSym = G.findDefinedSymbolByName(DSOBaseName)) + DSOBase = DSOBaseSym->getAddress(); + else + return make_error<StringError>("In " + G.getName() + + " could not find dso base symbol", + inconvertibleErrorCode()); + + using namespace shared; + using SPSRegisterArgs = + SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>, + SPSExecutorAddr, SPSExecutorAddrRange, SPSExecutorAddrRange>; + using SPSDeregisterArgs = + SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>; + + G.allocActions().push_back( + {cantFail(WrapperFunctionCall::Create<SPSRegisterArgs>( + Register, Instance, CodeRanges, DSOBase, EHFrameRange, + UnwindInfoRange)), + cantFail(WrapperFunctionCall::Create<SPSDeregisterArgs>( + Deregister, Instance, CodeRanges))}); + + return Error::success(); +} + +} // namespace llvm::orc |