aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/ExecutionEngine
diff options
context:
space:
mode:
authorLang Hames <lhames@gmail.com>2021-10-11 19:23:41 -0700
committerLang Hames <lhames@gmail.com>2021-10-11 19:23:41 -0700
commit6641d29b70993bce6dbd7e0e0f1040753d38842f (patch)
treefc16c9a66e305d125d428e65b98e5b9d31884a5d /llvm/lib/ExecutionEngine
parente50aea58d59c8cfae807a7fee21c4227472c0678 (diff)
downloadllvm-6641d29b70993bce6dbd7e0e0f1040753d38842f.zip
llvm-6641d29b70993bce6dbd7e0e0f1040753d38842f.tar.gz
llvm-6641d29b70993bce6dbd7e0e0f1040753d38842f.tar.bz2
Revert "[JITLink][ORC] Major JITLinkMemoryManager refactor."
This reverts commit e50aea58d59c8cfae807a7fee21c4227472c0678 while I investigate bot failures.
Diffstat (limited to 'llvm/lib/ExecutionEngine')
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt1
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h16
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp10
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp13
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp299
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h65
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp535
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp16
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp10
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp10
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/MemoryFlags.cpp33
-rw-r--r--llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp135
-rw-r--r--llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp2
-rw-r--r--llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp7
-rw-r--r--llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp196
-rw-r--r--llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp132
-rw-r--r--llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp3
-rw-r--r--llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp2
-rw-r--r--llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp27
-rw-r--r--llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp9
20 files changed, 666 insertions, 855 deletions
diff --git a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt
index 02157cb..4dfea9e 100644
--- a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt
@@ -3,7 +3,6 @@ add_llvm_component_library(LLVMJITLink
JITLink.cpp
JITLinkGeneric.cpp
JITLinkMemoryManager.cpp
- MemoryFlags.cpp
# Formats:
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
index fdc9877..05f84f3 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
@@ -36,9 +36,11 @@ protected:
}
Section &getCommonSection() {
- if (!CommonSection)
- CommonSection =
- &G->createSection(CommonSectionName, MemProt::Read | MemProt::Write);
+ if (!CommonSection) {
+ auto Prot = static_cast<sys::Memory::ProtectionFlags>(
+ sys::Memory::MF_READ | sys::Memory::MF_WRITE);
+ CommonSection = &G->createSection(CommonSectionName, Prot);
+ }
return *CommonSection;
}
@@ -293,11 +295,13 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySections() {
});
// Get the section's memory protection flags.
- MemProt Prot;
+ sys::Memory::ProtectionFlags Prot;
if (Sec.sh_flags & ELF::SHF_EXECINSTR)
- Prot = MemProt::Read | MemProt::Exec;
+ Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_EXEC);
else
- Prot = MemProt::Read | MemProt::Write;
+ Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_WRITE);
// For now we just use this to skip the "undefined" section, probably need
// to revist.
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp
index b057788..1f8dbad 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp
@@ -80,14 +80,16 @@ public:
private:
Section &getGOTSection() const {
if (!GOTSection)
- GOTSection = &G.createSection("$__GOT", MemProt::Read);
+ GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ);
return *GOTSection;
}
Section &getStubsSection() const {
- if (!StubsSection)
- StubsSection =
- &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec);
+ if (!StubsSection) {
+ auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
+ sys::Memory::MF_READ | sys::Memory::MF_EXEC);
+ StubsSection = &G.createSection("$__STUBS", StubsProt);
+ }
return *StubsSection;
}
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
index 79c6049..0a50f40 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
@@ -65,7 +65,8 @@ public:
Section &getTLSInfoSection() const {
if (!TLSInfoSection)
- TLSInfoSection = &G.createSection(ELFTLSInfoSectionName, MemProt::Read);
+ TLSInfoSection =
+ &G.createSection(ELFTLSInfoSectionName, sys::Memory::MF_READ);
return *TLSInfoSection;
}
@@ -171,14 +172,16 @@ public:
private:
Section &getGOTSection() const {
if (!GOTSection)
- GOTSection = &G.createSection(ELFGOTSectionName, MemProt::Read);
+ GOTSection = &G.createSection(ELFGOTSectionName, sys::Memory::MF_READ);
return *GOTSection;
}
Section &getStubsSection() const {
- if (!StubsSection)
- StubsSection =
- &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec);
+ if (!StubsSection) {
+ auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
+ sys::Memory::MF_READ | sys::Memory::MF_EXEC);
+ StubsSection = &G.createSection("$__STUBS", StubsProt);
+ }
return *StubsSection;
}
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
index 706688a..8bb61a2 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
@@ -48,21 +48,12 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
if (auto Err = runPasses(Passes.PostPrunePasses))
return Ctx->notifyFailed(std::move(Err));
- Ctx->getMemoryManager().allocate(
- Ctx->getJITLinkDylib(), *G,
- [S = std::move(Self)](AllocResult AR) mutable {
- auto *TmpSelf = S.get();
- TmpSelf->linkPhase2(std::move(S), std::move(AR));
- });
-}
+ // Sort blocks into segments.
+ auto Layout = layOutBlocks();
-void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
- AllocResult AR) {
-
- if (AR)
- Alloc = std::move(*AR);
- else
- return Ctx->notifyFailed(AR.takeError());
+ // Allocate memory for segments.
+ if (auto Err = allocateSegments(Layout))
+ return Ctx->notifyFailed(std::move(Err));
LLVM_DEBUG({
dbgs() << "Link graph \"" << G->getName()
@@ -82,16 +73,16 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
auto ExternalSymbols = getExternalSymbolNames();
- // If there are no external symbols then proceed immediately with phase 3.
+ // If there are no external symbols then proceed immediately with phase 2.
if (ExternalSymbols.empty()) {
LLVM_DEBUG({
dbgs() << "No external symbols for " << G->getName()
- << ". Proceeding immediately with link phase 3.\n";
+ << ". Proceeding immediately with link phase 2.\n";
});
// FIXME: Once callee expressions are defined to be sequenced before
// argument expressions (c++17) we can simplify this. See below.
auto &TmpSelf = *Self;
- TmpSelf.linkPhase3(std::move(Self), AsyncLookupResult());
+ TmpSelf.linkPhase2(std::move(Self), AsyncLookupResult(), std::move(Layout));
return;
}
@@ -109,31 +100,37 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
//
// Ctx->lookup(std::move(UnresolvedExternals),
// [Self=std::move(Self)](Expected<AsyncLookupResult> Result) {
- // Self->linkPhase3(std::move(Self), std::move(Result));
+ // Self->linkPhase2(std::move(Self), std::move(Result));
// });
- Ctx->lookup(std::move(ExternalSymbols),
- createLookupContinuation(
- [S = std::move(Self)](
- Expected<AsyncLookupResult> LookupResult) mutable {
- auto &TmpSelf = *S;
- TmpSelf.linkPhase3(std::move(S), std::move(LookupResult));
- }));
+ auto *TmpCtx = Ctx.get();
+ TmpCtx->lookup(std::move(ExternalSymbols),
+ createLookupContinuation(
+ [S = std::move(Self), L = std::move(Layout)](
+ Expected<AsyncLookupResult> LookupResult) mutable {
+ auto &TmpSelf = *S;
+ TmpSelf.linkPhase2(std::move(S), std::move(LookupResult),
+ std::move(L));
+ }));
}
-void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self,
- Expected<AsyncLookupResult> LR) {
+void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
+ Expected<AsyncLookupResult> LR,
+ SegmentLayoutMap Layout) {
LLVM_DEBUG({
- dbgs() << "Starting link phase 3 for graph " << G->getName() << "\n";
+ dbgs() << "Starting link phase 2 for graph " << G->getName() << "\n";
});
// If the lookup failed, bail out.
if (!LR)
- return abandonAllocAndBailOut(std::move(Self), LR.takeError());
+ return deallocateAndBailOut(LR.takeError());
// Assign addresses to external addressables.
applyLookupResult(*LR);
+ // Copy block content to working memory.
+ copyBlockContentToWorkingMemory(Layout, *Alloc);
+
LLVM_DEBUG({
dbgs() << "Link graph \"" << G->getName()
<< "\" before pre-fixup passes:\n";
@@ -141,7 +138,7 @@ void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self,
});
if (auto Err = runPasses(Passes.PreFixupPasses))
- return abandonAllocAndBailOut(std::move(Self), std::move(Err));
+ return deallocateAndBailOut(std::move(Err));
LLVM_DEBUG({
dbgs() << "Link graph \"" << G->getName() << "\" before copy-and-fixup:\n";
@@ -150,7 +147,7 @@ void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self,
// Fix up block content.
if (auto Err = fixUpBlocks(*G))
- return abandonAllocAndBailOut(std::move(Self), std::move(Err));
+ return deallocateAndBailOut(std::move(Err));
LLVM_DEBUG({
dbgs() << "Link graph \"" << G->getName() << "\" after copy-and-fixup:\n";
@@ -158,25 +155,27 @@ void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self,
});
if (auto Err = runPasses(Passes.PostFixupPasses))
- return abandonAllocAndBailOut(std::move(Self), std::move(Err));
+ return deallocateAndBailOut(std::move(Err));
- Alloc->finalize([S = std::move(Self)](FinalizeResult FR) mutable {
- auto *TmpSelf = S.get();
- TmpSelf->linkPhase4(std::move(S), std::move(FR));
- });
+ // FIXME: Use move capture once we have c++14.
+ auto *UnownedSelf = Self.release();
+ auto Phase3Continuation = [UnownedSelf](Error Err) {
+ std::unique_ptr<JITLinkerBase> Self(UnownedSelf);
+ UnownedSelf->linkPhase3(std::move(Self), std::move(Err));
+ };
+
+ Alloc->finalizeAsync(std::move(Phase3Continuation));
}
-void JITLinkerBase::linkPhase4(std::unique_ptr<JITLinkerBase> Self,
- FinalizeResult FR) {
+void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err) {
LLVM_DEBUG({
- dbgs() << "Starting link phase 4 for graph " << G->getName() << "\n";
+ dbgs() << "Starting link phase 3 for graph " << G->getName() << "\n";
});
- if (!FR)
- return Ctx->notifyFailed(FR.takeError());
-
- Ctx->notifyFinalized(std::move(*FR));
+ if (Err)
+ return deallocateAndBailOut(std::move(Err));
+ Ctx->notifyFinalized(std::move(Alloc));
LLVM_DEBUG({ dbgs() << "Link of graph " << G->getName() << " complete\n"; });
}
@@ -188,6 +187,131 @@ Error JITLinkerBase::runPasses(LinkGraphPassList &Passes) {
return Error::success();
}
+JITLinkerBase::SegmentLayoutMap JITLinkerBase::layOutBlocks() {
+
+ SegmentLayoutMap Layout;
+
+ /// Partition blocks based on permissions and content vs. zero-fill.
+ for (auto *B : G->blocks()) {
+ auto &SegLists = Layout[B->getSection().getProtectionFlags()];
+ if (!B->isZeroFill())
+ SegLists.ContentBlocks.push_back(B);
+ else
+ SegLists.ZeroFillBlocks.push_back(B);
+ }
+
+ /// Sort blocks within each list.
+ for (auto &KV : Layout) {
+
+ auto CompareBlocks = [](const Block *LHS, const Block *RHS) {
+ // Sort by section, address and size
+ if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal())
+ return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal();
+ if (LHS->getAddress() != RHS->getAddress())
+ return LHS->getAddress() < RHS->getAddress();
+ return LHS->getSize() < RHS->getSize();
+ };
+
+ auto &SegLists = KV.second;
+ llvm::sort(SegLists.ContentBlocks, CompareBlocks);
+ llvm::sort(SegLists.ZeroFillBlocks, CompareBlocks);
+ }
+
+ LLVM_DEBUG({
+ dbgs() << "Computed segment ordering:\n";
+ for (auto &KV : Layout) {
+ dbgs() << " Segment "
+ << static_cast<sys::Memory::ProtectionFlags>(KV.first) << ":\n";
+ auto &SL = KV.second;
+ for (auto &SIEntry :
+ {std::make_pair(&SL.ContentBlocks, "content block"),
+ std::make_pair(&SL.ZeroFillBlocks, "zero-fill block")}) {
+ dbgs() << " " << SIEntry.second << ":\n";
+ for (auto *B : *SIEntry.first)
+ dbgs() << " " << *B << "\n";
+ }
+ }
+ });
+
+ return Layout;
+}
+
+Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) {
+
+ // Compute segment sizes and allocate memory.
+ LLVM_DEBUG(dbgs() << "JIT linker requesting: { ");
+ JITLinkMemoryManager::SegmentsRequestMap Segments;
+ for (auto &KV : Layout) {
+ auto &Prot = KV.first;
+ auto &SegLists = KV.second;
+
+ uint64_t SegAlign = 1;
+
+ // Calculate segment content size.
+ size_t SegContentSize = 0;
+ for (auto *B : SegLists.ContentBlocks) {
+ SegAlign = std::max(SegAlign, B->getAlignment());
+ SegContentSize = alignToBlock(SegContentSize, *B);
+ SegContentSize += B->getSize();
+ }
+
+ uint64_t SegZeroFillStart = SegContentSize;
+ uint64_t SegZeroFillEnd = SegZeroFillStart;
+
+ for (auto *B : SegLists.ZeroFillBlocks) {
+ SegAlign = std::max(SegAlign, B->getAlignment());
+ SegZeroFillEnd = alignToBlock(SegZeroFillEnd, *B);
+ SegZeroFillEnd += B->getSize();
+ }
+
+ Segments[Prot] = {SegAlign, SegContentSize,
+ SegZeroFillEnd - SegZeroFillStart};
+
+ LLVM_DEBUG({
+ dbgs() << (&KV == &*Layout.begin() ? "" : "; ")
+ << static_cast<sys::Memory::ProtectionFlags>(Prot)
+ << ": alignment = " << SegAlign
+ << ", content size = " << SegContentSize
+ << ", zero-fill size = " << (SegZeroFillEnd - SegZeroFillStart);
+ });
+ }
+ LLVM_DEBUG(dbgs() << " }\n");
+
+ if (auto AllocOrErr =
+ Ctx->getMemoryManager().allocate(Ctx->getJITLinkDylib(), Segments))
+ Alloc = std::move(*AllocOrErr);
+ else
+ return AllocOrErr.takeError();
+
+ LLVM_DEBUG({
+ dbgs() << "JIT linker got memory (working -> target):\n";
+ for (auto &KV : Layout) {
+ auto Prot = static_cast<sys::Memory::ProtectionFlags>(KV.first);
+ dbgs() << " " << Prot << ": "
+ << (const void *)Alloc->getWorkingMemory(Prot).data() << " -> "
+ << formatv("{0:x16}", Alloc->getTargetMemory(Prot)) << "\n";
+ }
+ });
+
+ // Update block target addresses.
+ for (auto &KV : Layout) {
+ auto &Prot = KV.first;
+ auto &SL = KV.second;
+
+ JITTargetAddress NextBlockAddr =
+ Alloc->getTargetMemory(static_cast<sys::Memory::ProtectionFlags>(Prot));
+
+ for (auto *SIList : {&SL.ContentBlocks, &SL.ZeroFillBlocks})
+ for (auto *B : *SIList) {
+ NextBlockAddr = alignToBlock(NextBlockAddr, *B);
+ B->setAddress(NextBlockAddr);
+ NextBlockAddr += B->getSize();
+ }
+ }
+
+ return Error::success();
+}
+
JITLinkContext::LookupMap JITLinkerBase::getExternalSymbolNames() const {
// Identify unresolved external symbols.
JITLinkContext::LookupMap UnresolvedExternals;
@@ -227,13 +351,90 @@ void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) {
});
}
-void JITLinkerBase::abandonAllocAndBailOut(std::unique_ptr<JITLinkerBase> Self,
- Error Err) {
+void JITLinkerBase::copyBlockContentToWorkingMemory(
+ const SegmentLayoutMap &Layout, JITLinkMemoryManager::Allocation &Alloc) {
+
+ LLVM_DEBUG(dbgs() << "Copying block content:\n");
+ for (auto &KV : Layout) {
+ auto &Prot = KV.first;
+ auto &SegLayout = KV.second;
+
+ auto SegMem =
+ Alloc.getWorkingMemory(static_cast<sys::Memory::ProtectionFlags>(Prot));
+
+ LLVM_DEBUG({
+ dbgs() << " Processing segment "
+ << static_cast<sys::Memory::ProtectionFlags>(Prot) << " [ "
+ << (const void *)SegMem.data() << " .. "
+ << (const void *)((char *)SegMem.data() + SegMem.size())
+ << " ]\n Processing content sections:\n";
+ });
+
+ if (SegLayout.ContentBlocks.empty()) {
+ LLVM_DEBUG(dbgs() << " No content blocks.\n");
+ continue;
+ }
+
+ size_t BlockOffset = 0;
+ size_t LastBlockEnd = 0;
+
+ for (auto *B : SegLayout.ContentBlocks) {
+ LLVM_DEBUG(dbgs() << " " << *B << ":\n");
+
+ // Pad to alignment/alignment-offset.
+ BlockOffset = alignToBlock(BlockOffset, *B);
+
+ LLVM_DEBUG({
+ dbgs() << " Bumped block offset to "
+ << formatv("{0:x}", BlockOffset) << " to meet block alignment "
+ << B->getAlignment() << " and alignment offset "
+ << B->getAlignmentOffset() << "\n";
+ });
+
+ // Zero pad up to alignment.
+ LLVM_DEBUG({
+ if (LastBlockEnd != BlockOffset)
+ dbgs() << " Zero padding from " << formatv("{0:x}", LastBlockEnd)
+ << " to " << formatv("{0:x}", BlockOffset) << "\n";
+ });
+
+ for (; LastBlockEnd != BlockOffset; ++LastBlockEnd)
+ *(SegMem.data() + LastBlockEnd) = 0;
+
+ // Copy initial block content.
+ LLVM_DEBUG({
+ dbgs() << " Copying block " << *B << " content, "
+ << B->getContent().size() << " bytes, from "
+ << (const void *)B->getContent().data() << " to offset "
+ << formatv("{0:x}", BlockOffset) << "\n";
+ });
+ memcpy(SegMem.data() + BlockOffset, B->getContent().data(),
+ B->getContent().size());
+
+ // Point the block's content to the fixed up buffer.
+ B->setMutableContent(
+ {SegMem.data() + BlockOffset, B->getContent().size()});
+
+ // Update block end pointer.
+ LastBlockEnd = BlockOffset + B->getContent().size();
+ BlockOffset = LastBlockEnd;
+ }
+
+ // Zero pad the rest of the segment.
+ LLVM_DEBUG({
+ dbgs() << " Zero padding end of segment from offset "
+ << formatv("{0:x}", LastBlockEnd) << " to "
+ << formatv("{0:x}", SegMem.size()) << "\n";
+ });
+ for (; LastBlockEnd != SegMem.size(); ++LastBlockEnd)
+ *(SegMem.data() + LastBlockEnd) = 0;
+ }
+}
+
+void JITLinkerBase::deallocateAndBailOut(Error Err) {
assert(Err && "Should not be bailing out on success value");
- assert(Alloc && "can not call abandonAllocAndBailOut before allocation");
- Alloc->abandon([S = std::move(Self), E1 = std::move(Err)](Error E2) mutable {
- S->Ctx->notifyFailed(joinErrors(std::move(E1), std::move(E2)));
- });
+ assert(Alloc && "can not call deallocateAndBailOut before allocation");
+ Ctx->notifyFailed(joinErrors(std::move(Err), Alloc->deallocate()));
}
void prune(LinkGraph &G) {
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
index e4fdda0..6b815fe 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
@@ -42,9 +42,14 @@ public:
virtual ~JITLinkerBase();
protected:
- using InFlightAlloc = JITLinkMemoryManager::InFlightAlloc;
- using AllocResult = Expected<std::unique_ptr<InFlightAlloc>>;
- using FinalizeResult = Expected<JITLinkMemoryManager::FinalizedAlloc>;
+ struct SegmentLayout {
+ using BlocksList = std::vector<Block *>;
+
+ BlocksList ContentBlocks;
+ BlocksList ZeroFillBlocks;
+ };
+
+ using SegmentLayoutMap = DenseMap<unsigned, SegmentLayout>;
// Returns the PassConfiguration for this instance. This can be used by
// JITLinkerBase implementations to add late passes that reference their
@@ -56,27 +61,39 @@ protected:
// 1.1: Run pre-prune passes
// 1.2: Prune graph
// 1.3: Run post-prune passes
- // 1.4: Allocate memory.
+ // 1.4: Sort blocks into segments
+ // 1.5: Allocate segment memory, update node vmaddrs to target vmaddrs
+ // 1.6: Run post-allocation passes
+ // 1.7: Notify context of final assigned symbol addresses
+ // 1.8: Identify external symbols and make an async call to resolve
void linkPhase1(std::unique_ptr<JITLinkerBase> Self);
// Phase 2:
- // 2.2: Run post-allocation passes
- // 2.3: Notify context of final assigned symbol addresses
- // 2.4: Identify external symbols and make an async call to resolve
- void linkPhase2(std::unique_ptr<JITLinkerBase> Self, AllocResult AR);
+ // 2.1: Apply resolution results
+ // 2.2: Run pre-fixup passes
+ // 2.3: Fix up block contents
+ // 2.4: Run post-fixup passes
+ // 2.5: Make an async call to transfer and finalize memory.
+ void linkPhase2(std::unique_ptr<JITLinkerBase> Self,
+ Expected<AsyncLookupResult> LookupResult,
+ SegmentLayoutMap Layout);
// Phase 3:
- // 3.1: Apply resolution results
- // 3.2: Run pre-fixup passes
- // 3.3: Fix up block contents
- // 3.4: Run post-fixup passes
- // 3.5: Make an async call to transfer and finalize memory.
- void linkPhase3(std::unique_ptr<JITLinkerBase> Self,
- Expected<AsyncLookupResult> LookupResult);
-
- // Phase 4:
- // 4.1: Call OnFinalized callback, handing off allocation.
- void linkPhase4(std::unique_ptr<JITLinkerBase> Self, FinalizeResult FR);
+ // 3.1: Call OnFinalized callback, handing off allocation.
+ void linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err);
+
+ // Align a JITTargetAddress to conform with block alignment requirements.
+ static JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) {
+ uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment();
+ return Addr + Delta;
+ }
+
+ // Align a pointer to conform with block alignment requirements.
+ static char *alignToBlock(char *P, Block &B) {
+ uint64_t PAddr = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(P));
+ uint64_t Delta = (B.getAlignmentOffset() - PAddr) % B.getAlignment();
+ return P + Delta;
+ }
private:
// Run all passes in the given pass list, bailing out immediately if any pass
@@ -87,14 +104,18 @@ private:
// Implemented in JITLinker.
virtual Error fixUpBlocks(LinkGraph &G) const = 0;
+ SegmentLayoutMap layOutBlocks();
+ Error allocateSegments(const SegmentLayoutMap &Layout);
JITLinkContext::LookupMap getExternalSymbolNames() const;
void applyLookupResult(AsyncLookupResult LR);
- void abandonAllocAndBailOut(std::unique_ptr<JITLinkerBase> Self, Error Err);
+ void copyBlockContentToWorkingMemory(const SegmentLayoutMap &Layout,
+ JITLinkMemoryManager::Allocation &Alloc);
+ void deallocateAndBailOut(Error Err);
std::unique_ptr<JITLinkContext> Ctx;
std::unique_ptr<LinkGraph> G;
PassConfiguration Passes;
- std::unique_ptr<InFlightAlloc> Alloc;
+ std::unique_ptr<JITLinkMemoryManager::Allocation> Alloc;
};
template <typename LinkerImpl> class JITLinker : public JITLinkerBase {
@@ -131,8 +152,6 @@ private:
// Copy Block data and apply fixups.
LLVM_DEBUG(dbgs() << " Applying fixups.\n");
- assert((!B->isZeroFill() || B->edges_size() == 0) &&
- "Edges in zero-fill block?");
for (auto &E : B->edges()) {
// Skip non-relocation edges.
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
index 9069ad8..d572a0b 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
@@ -7,479 +7,128 @@
//===----------------------------------------------------------------------===//
#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
-#include "llvm/ExecutionEngine/JITLink/JITLink.h"
-#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/Process.h"
-#define DEBUG_TYPE "jitlink"
-
namespace llvm {
namespace jitlink {
JITLinkMemoryManager::~JITLinkMemoryManager() = default;
-JITLinkMemoryManager::InFlightAlloc::~InFlightAlloc() = default;
-
-static Error runAllocAction(JITLinkMemoryManager::AllocActionCall &C) {
- using DeallocFnTy = char *(*)(const void *, size_t);
- auto *Fn = jitTargetAddressToPointer<DeallocFnTy>(C.FnAddr);
-
- if (char *ErrMsg = Fn(jitTargetAddressToPointer<const void *>(C.CtxAddr),
- static_cast<size_t>(C.CtxSize))) {
- auto E = make_error<StringError>(ErrMsg, inconvertibleErrorCode());
- free(ErrMsg);
- return E;
- }
-
- return Error::success();
-}
-
-// Align a JITTargetAddress to conform with block alignment requirements.
-static JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) {
- uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment();
- return Addr + Delta;
-}
-
-BasicLayout::BasicLayout(LinkGraph &G) : G(G) {
-
- for (auto &Sec : G.sections()) {
- // Skip empty sections.
- if (empty(Sec.blocks()))
- continue;
-
- auto &Seg = Segments[{Sec.getMemProt(), Sec.getMemDeallocPolicy()}];
- for (auto *B : Sec.blocks())
- if (LLVM_LIKELY(!B->isZeroFill()))
- Seg.ContentBlocks.push_back(B);
- else
- Seg.ZeroFillBlocks.push_back(B);
- }
-
- // Build Segments map.
- auto CompareBlocks = [](const Block *LHS, const Block *RHS) {
- // Sort by section, address and size
- if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal())
- return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal();
- if (LHS->getAddress() != RHS->getAddress())
- return LHS->getAddress() < RHS->getAddress();
- return LHS->getSize() < RHS->getSize();
- };
-
- LLVM_DEBUG(dbgs() << "Generated BasicLayout for " << G.getName() << ":\n");
- for (auto &KV : Segments) {
- auto &Seg = KV.second;
-
- llvm::sort(Seg.ContentBlocks, CompareBlocks);
- llvm::sort(Seg.ZeroFillBlocks, CompareBlocks);
-
- for (auto *B : Seg.ContentBlocks) {
- Seg.ContentSize = alignToBlock(Seg.ContentSize, *B);
- Seg.ContentSize += B->getSize();
- Seg.Alignment = std::max(Seg.Alignment, Align(B->getAlignment()));
- }
-
- uint64_t SegEndOffset = Seg.ContentSize;
- for (auto *B : Seg.ZeroFillBlocks) {
- SegEndOffset = alignToBlock(SegEndOffset, *B);
- SegEndOffset += B->getSize();
- Seg.Alignment = std::max(Seg.Alignment, Align(B->getAlignment()));
- }
- Seg.ZeroFillSize = SegEndOffset - Seg.ContentSize;
-
- LLVM_DEBUG({
- dbgs() << " Seg " << KV.first
- << ": content-size=" << formatv("{0:x}", Seg.ContentSize)
- << ", zero-fill-size=" << formatv("{0:x}", Seg.ZeroFillSize)
- << ", align=" << formatv("{0:x}", Seg.Alignment.value()) << "\n";
- });
- }
-}
-
-Expected<BasicLayout::ContiguousPageBasedLayoutSizes>
-BasicLayout::getContiguousPageBasedLayoutSizes(uint64_t PageSize) {
- ContiguousPageBasedLayoutSizes SegsSizes;
-
- for (auto &KV : segments()) {
- auto &AG = KV.first;
- auto &Seg = KV.second;
-
- if (Seg.Alignment > PageSize)
- return make_error<StringError>("Segment alignment greater than page size",
- inconvertibleErrorCode());
-
- uint64_t SegSize = alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize);
- if (AG.getMemDeallocPolicy() == MemDeallocPolicy::Standard)
- SegsSizes.StandardSegs += SegSize;
- else
- SegsSizes.FinalizeSegs += SegSize;
- }
-
- return SegsSizes;
-}
-
-Error BasicLayout::apply() {
- for (auto &KV : Segments) {
- auto &Seg = KV.second;
-
- assert(!(Seg.ContentBlocks.empty() && Seg.ZeroFillBlocks.empty()) &&
- "Empty section recorded?");
-
- for (auto *B : Seg.ContentBlocks) {
- // Align addr and working-mem-offset.
- Seg.Addr = alignToBlock(Seg.Addr, *B);
- Seg.NextWorkingMemOffset = alignToBlock(Seg.NextWorkingMemOffset, *B);
-
- // Update block addr.
- B->setAddress(Seg.Addr);
- Seg.Addr += B->getSize();
-
- // Copy content to working memory, then update content to point at working
- // memory.
- memcpy(Seg.WorkingMem + Seg.NextWorkingMemOffset, B->getContent().data(),
- B->getSize());
- B->setMutableContent(
- {Seg.WorkingMem + Seg.NextWorkingMemOffset, B->getSize()});
- Seg.NextWorkingMemOffset += B->getSize();
- }
-
- for (auto *B : Seg.ZeroFillBlocks) {
- // Align addr.
- Seg.Addr = alignToBlock(Seg.Addr, *B);
- // Update block addr.
- B->setAddress(Seg.Addr);
- Seg.Addr += B->getSize();
- }
-
- Seg.ContentBlocks.clear();
- Seg.ZeroFillBlocks.clear();
- }
-
- return Error::success();
-}
-
-JITLinkMemoryManager::AllocActions &BasicLayout::graphAllocActions() {
- return G.allocActions();
-}
-
-void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr,
- const JITLinkDylib *JD, SegmentMap Segments,
- OnCreatedFunction OnCreated) {
-
- static_assert(AllocGroup::NumGroups == 16,
- "AllocGroup has changed. Section names below must be updated");
- StringRef AGSectionNames[] = {
- "__---.standard", "__R--.standard", "__-W-.standard", "__RW-.standard",
- "__--X.standard", "__R-X.standard", "__-WX.standard", "__RWX.standard",
- "__---.finalize", "__R--.finalize", "__-W-.finalize", "__RW-.finalize",
- "__--X.finalize", "__R-X.finalize", "__-WX.finalize", "__RWX.finalize"};
-
- auto G =
- std::make_unique<LinkGraph>("", Triple(), 0, support::native, nullptr);
- AllocGroupSmallMap<Block *> ContentBlocks;
-
- JITTargetAddress NextAddr = 0x100000;
- for (auto &KV : Segments) {
- auto &AG = KV.first;
- auto &Seg = KV.second;
-
- auto AGSectionName =
- AGSectionNames[static_cast<unsigned>(AG.getMemProt()) |
- static_cast<bool>(AG.getMemDeallocPolicy()) << 3];
-
- auto &Sec = G->createSection(AGSectionName, AG.getMemProt());
- Sec.setMemDeallocPolicy(AG.getMemDeallocPolicy());
-
- if (Seg.ContentSize != 0) {
- NextAddr = alignTo(NextAddr, Seg.ContentAlign);
- auto &B =
- G->createMutableContentBlock(Sec, G->allocateBuffer(Seg.ContentSize),
- NextAddr, Seg.ContentAlign.value(), 0);
- ContentBlocks[AG] = &B;
- NextAddr += Seg.ContentSize;
+JITLinkMemoryManager::Allocation::~Allocation() = default;
+
+Expected<std::unique_ptr<JITLinkMemoryManager::Allocation>>
+InProcessMemoryManager::allocate(const JITLinkDylib *JD,
+ const SegmentsRequestMap &Request) {
+
+ using AllocationMap = DenseMap<unsigned, sys::MemoryBlock>;
+
+ // Local class for allocation.
+ class IPMMAlloc : public Allocation {
+ public:
+ IPMMAlloc(AllocationMap SegBlocks) : SegBlocks(std::move(SegBlocks)) {}
+ MutableArrayRef<char> getWorkingMemory(ProtectionFlags Seg) override {
+ assert(SegBlocks.count(Seg) && "No allocation for segment");
+ return {static_cast<char *>(SegBlocks[Seg].base()),
+ SegBlocks[Seg].allocatedSize()};
}
- }
-
- // GRef declared separately since order-of-argument-eval isn't specified.
- auto &GRef = *G;
- MemMgr.allocate(JD, GRef,
- [G = std::move(G), ContentBlocks = std::move(ContentBlocks),
- OnCreated = std::move(OnCreated)](
- JITLinkMemoryManager::AllocResult Alloc) mutable {
- if (!Alloc)
- OnCreated(Alloc.takeError());
- else
- OnCreated(SimpleSegmentAlloc(std::move(G),
- std::move(ContentBlocks),
- std::move(*Alloc)));
- });
-}
-
-Expected<SimpleSegmentAlloc>
-SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD,
- SegmentMap Segments) {
- std::promise<MSVCPExpected<SimpleSegmentAlloc>> AllocP;
- auto AllocF = AllocP.get_future();
- Create(MemMgr, JD, std::move(Segments),
- [&](Expected<SimpleSegmentAlloc> Result) {
- AllocP.set_value(std::move(Result));
- });
- return AllocF.get();
-}
-
-SimpleSegmentAlloc::SimpleSegmentAlloc(SimpleSegmentAlloc &&) = default;
-SimpleSegmentAlloc &
-SimpleSegmentAlloc::operator=(SimpleSegmentAlloc &&) = default;
-SimpleSegmentAlloc::~SimpleSegmentAlloc() {}
-
-SimpleSegmentAlloc::SegmentInfo SimpleSegmentAlloc::getSegInfo(AllocGroup AG) {
- auto I = ContentBlocks.find(AG);
- if (I != ContentBlocks.end()) {
- auto &B = *I->second;
- return {B.getAddress(), B.getAlreadyMutableContent()};
- }
- return {};
-}
-
-SimpleSegmentAlloc::SimpleSegmentAlloc(
- std::unique_ptr<LinkGraph> G, AllocGroupSmallMap<Block *> ContentBlocks,
- std::unique_ptr<JITLinkMemoryManager::InFlightAlloc> Alloc)
- : G(std::move(G)), ContentBlocks(std::move(ContentBlocks)),
- Alloc(std::move(Alloc)) {}
-
-class InProcessMemoryManager::IPInFlightAlloc
- : public JITLinkMemoryManager::InFlightAlloc {
-public:
- IPInFlightAlloc(InProcessMemoryManager &MemMgr, LinkGraph &G, BasicLayout BL,
- sys::MemoryBlock StandardSegments,
- sys::MemoryBlock FinalizationSegments)
- : MemMgr(MemMgr), G(G), BL(std::move(BL)),
- StandardSegments(std::move(StandardSegments)),
- FinalizationSegments(std::move(FinalizationSegments)) {}
-
- void finalize(OnFinalizedFunction OnFinalized) override {
-
- // Apply memory protections to all segments.
- if (auto Err = applyProtections()) {
- OnFinalized(std::move(Err));
- return;
- }
-
- // Run finalization actions.
- // FIXME: Roll back previous successful actions on failure.
- std::vector<AllocActionCall> DeallocActions;
- DeallocActions.reserve(G.allocActions().size());
- for (auto &ActPair : G.allocActions()) {
- if (ActPair.Finalize.FnAddr)
- if (auto Err = runAllocAction(ActPair.Finalize)) {
- OnFinalized(std::move(Err));
- return;
- }
- if (ActPair.Dealloc.FnAddr)
- DeallocActions.push_back(ActPair.Dealloc);
+ JITTargetAddress getTargetMemory(ProtectionFlags Seg) override {
+ assert(SegBlocks.count(Seg) && "No allocation for segment");
+ return pointerToJITTargetAddress(SegBlocks[Seg].base());
}
- G.allocActions().clear();
-
- // Release the finalize segments slab.
- if (auto EC = sys::Memory::releaseMappedMemory(FinalizationSegments)) {
- OnFinalized(errorCodeToError(EC));
- return;
+ void finalizeAsync(FinalizeContinuation OnFinalize) override {
+ OnFinalize(applyProtections());
}
-
- // Continue with finalized allocation.
- OnFinalized(MemMgr.createFinalizedAlloc(std::move(StandardSegments),
- std::move(DeallocActions)));
- }
-
- void abandon(OnAbandonedFunction OnAbandoned) override {
- Error Err = Error::success();
- if (auto EC = sys::Memory::releaseMappedMemory(FinalizationSegments))
- Err = joinErrors(std::move(Err), errorCodeToError(EC));
- if (auto EC = sys::Memory::releaseMappedMemory(StandardSegments))
- Err = joinErrors(std::move(Err), errorCodeToError(EC));
- OnAbandoned(std::move(Err));
- }
-
-private:
- Error applyProtections() {
- for (auto &KV : BL.segments()) {
- const auto &AG = KV.first;
- auto &Seg = KV.second;
-
- auto Prot = toSysMemoryProtectionFlags(AG.getMemProt());
-
- uint64_t SegSize =
- alignTo(Seg.ContentSize + Seg.ZeroFillSize, MemMgr.PageSize);
- sys::MemoryBlock MB(Seg.WorkingMem, SegSize);
- if (auto EC = sys::Memory::protectMappedMemory(MB, Prot))
+ Error deallocate() override {
+ if (SegBlocks.empty())
+ return Error::success();
+ void *SlabStart = SegBlocks.begin()->second.base();
+ char *SlabEnd = (char *)SlabStart;
+ for (auto &KV : SegBlocks) {
+ SlabStart = std::min(SlabStart, KV.second.base());
+ SlabEnd = std::max(SlabEnd, (char *)(KV.second.base()) +
+ KV.second.allocatedSize());
+ }
+ size_t SlabSize = SlabEnd - (char *)SlabStart;
+ assert((SlabSize % sys::Process::getPageSizeEstimate()) == 0 &&
+ "Slab size is not a multiple of page size");
+ sys::MemoryBlock Slab(SlabStart, SlabSize);
+ if (auto EC = sys::Memory::releaseMappedMemory(Slab))
return errorCodeToError(EC);
- if (Prot & sys::Memory::MF_EXEC)
- sys::Memory::InvalidateInstructionCache(MB.base(), MB.allocatedSize());
+ return Error::success();
}
- return Error::success();
- }
-
- InProcessMemoryManager &MemMgr;
- LinkGraph &G;
- BasicLayout BL;
- sys::MemoryBlock StandardSegments;
- sys::MemoryBlock FinalizationSegments;
-};
-
-Expected<std::unique_ptr<InProcessMemoryManager>>
-InProcessMemoryManager::Create() {
- if (auto PageSize = sys::Process::getPageSize())
- return std::make_unique<InProcessMemoryManager>(*PageSize);
- else
- return PageSize.takeError();
-}
-
-void InProcessMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G,
- OnAllocatedFunction OnAllocated) {
-
- // FIXME: Just check this once on startup.
- if (!isPowerOf2_64((uint64_t)PageSize)) {
- OnAllocated(make_error<StringError>("Page size is not a power of 2",
- inconvertibleErrorCode()));
- return;
- }
- BasicLayout BL(G);
-
- /// Scan the request and calculate the group and total sizes.
- /// Check that segment size is no larger than a page.
- auto SegsSizes = BL.getContiguousPageBasedLayoutSizes(PageSize);
- if (!SegsSizes) {
- OnAllocated(SegsSizes.takeError());
- return;
- }
-
- // Allocate one slab for the whole thing (to make sure everything is
- // in-range), then partition into standard and finalization blocks.
- //
- // FIXME: Make two separate allocations in the future to reduce
- // fragmentation: finalization segments will usually be a single page, and
- // standard segments are likely to be more than one page. Where multiple
- // allocations are in-flight at once (likely) the current approach will leave
- // a lot of single-page holes.
- sys::MemoryBlock Slab;
- sys::MemoryBlock StandardSegsMem;
- sys::MemoryBlock FinalizeSegsMem;
- {
- const sys::Memory::ProtectionFlags ReadWrite =
- static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
- sys::Memory::MF_WRITE);
-
- std::error_code EC;
- Slab = sys::Memory::allocateMappedMemory(SegsSizes->total(), nullptr,
- ReadWrite, EC);
-
- if (EC) {
- OnAllocated(errorCodeToError(EC));
- return;
+ private:
+ Error applyProtections() {
+ for (auto &KV : SegBlocks) {
+ auto &Prot = KV.first;
+ auto &Block = KV.second;
+ if (auto EC = sys::Memory::protectMappedMemory(Block, Prot))
+ return errorCodeToError(EC);
+ if (Prot & sys::Memory::MF_EXEC)
+ sys::Memory::InvalidateInstructionCache(Block.base(),
+ Block.allocatedSize());
+ }
+ return Error::success();
}
- // Zero-fill the whole slab up-front.
- memset(Slab.base(), 0, Slab.allocatedSize());
-
- StandardSegsMem = {Slab.base(), SegsSizes->StandardSegs};
- FinalizeSegsMem = {(void *)((char *)Slab.base() + SegsSizes->StandardSegs),
- SegsSizes->FinalizeSegs};
- }
-
- auto NextStandardSegAddr = pointerToJITTargetAddress(StandardSegsMem.base());
- auto NextFinalizeSegAddr = pointerToJITTargetAddress(FinalizeSegsMem.base());
-
- LLVM_DEBUG({
- dbgs() << "InProcessMemoryManager allocated:\n";
- if (SegsSizes->StandardSegs)
- dbgs() << formatv(" [ {0:x16} -- {1:x16} ]", NextStandardSegAddr,
- NextStandardSegAddr + StandardSegsMem.allocatedSize())
- << " to stardard segs\n";
- else
- dbgs() << " no standard segs\n";
- if (SegsSizes->FinalizeSegs)
- dbgs() << formatv(" [ {0:x16} -- {1:x16} ]", NextFinalizeSegAddr,
- NextFinalizeSegAddr + FinalizeSegsMem.allocatedSize())
- << " to finalize segs\n";
- else
- dbgs() << " no finalize segs\n";
- });
+ AllocationMap SegBlocks;
+ };
- // Build ProtMap, assign addresses.
- for (auto &KV : BL.segments()) {
- auto &AG = KV.first;
- auto &Seg = KV.second;
+ if (!isPowerOf2_64((uint64_t)sys::Process::getPageSizeEstimate()))
+ return make_error<StringError>("Page size is not a power of 2",
+ inconvertibleErrorCode());
- auto &SegAddr = (AG.getMemDeallocPolicy() == MemDeallocPolicy::Standard)
- ? NextStandardSegAddr
- : NextFinalizeSegAddr;
+ AllocationMap Blocks;
+ const sys::Memory::ProtectionFlags ReadWrite =
+ static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_WRITE);
- Seg.WorkingMem = jitTargetAddressToPointer<char *>(SegAddr);
- Seg.Addr = SegAddr;
+ // Compute the total number of pages to allocate.
+ size_t TotalSize = 0;
+ for (auto &KV : Request) {
+ const auto &Seg = KV.second;
- SegAddr += alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize);
- }
+ if (Seg.getAlignment() > sys::Process::getPageSizeEstimate())
+ return make_error<StringError>("Cannot request higher than page "
+ "alignment",
+ inconvertibleErrorCode());
- if (auto Err = BL.apply()) {
- OnAllocated(std::move(Err));
- return;
+ TotalSize = alignTo(TotalSize, sys::Process::getPageSizeEstimate());
+ TotalSize += Seg.getContentSize();
+ TotalSize += Seg.getZeroFillSize();
}
- OnAllocated(std::make_unique<IPInFlightAlloc>(*this, G, std::move(BL),
- std::move(StandardSegsMem),
- std::move(FinalizeSegsMem)));
-}
+ // Allocate one slab to cover all the segments.
+ std::error_code EC;
+ auto SlabRemaining =
+ sys::Memory::allocateMappedMemory(TotalSize, nullptr, ReadWrite, EC);
-void InProcessMemoryManager::deallocate(std::vector<FinalizedAlloc> Allocs,
- OnDeallocatedFunction OnDeallocated) {
- std::vector<sys::MemoryBlock> StandardSegmentsList;
- std::vector<std::vector<AllocActionCall>> DeallocActionsList;
+ if (EC)
+ return errorCodeToError(EC);
- {
- std::lock_guard<std::mutex> Lock(FinalizedAllocsMutex);
- for (auto &Alloc : Allocs) {
- auto *FA =
- jitTargetAddressToPointer<FinalizedAllocInfo *>(Alloc.release());
- StandardSegmentsList.push_back(std::move(FA->StandardSegments));
- if (!FA->DeallocActions.empty())
- DeallocActionsList.push_back(std::move(FA->DeallocActions));
- FA->~FinalizedAllocInfo();
- FinalizedAllocInfos.Deallocate(FA);
- }
- }
+ // Allocate segment memory from the slab.
+ for (auto &KV : Request) {
- Error DeallocErr = Error::success();
+ const auto &Seg = KV.second;
- while (!DeallocActionsList.empty()) {
- auto &DeallocActions = DeallocActionsList.back();
- auto &StandardSegments = StandardSegmentsList.back();
+ uint64_t SegmentSize = alignTo(Seg.getContentSize() + Seg.getZeroFillSize(),
+ sys::Process::getPageSizeEstimate());
+ assert(SlabRemaining.allocatedSize() >= SegmentSize &&
+ "Mapping exceeds allocation");
- /// Run any deallocate calls.
- while (!DeallocActions.empty()) {
- if (auto Err = runAllocAction(DeallocActions.back()))
- DeallocErr = joinErrors(std::move(DeallocErr), std::move(Err));
- DeallocActions.pop_back();
- }
+ sys::MemoryBlock SegMem(SlabRemaining.base(), SegmentSize);
+ SlabRemaining = sys::MemoryBlock((char *)SlabRemaining.base() + SegmentSize,
+ SlabRemaining.allocatedSize() - SegmentSize);
- /// Release the standard segments slab.
- if (auto EC = sys::Memory::releaseMappedMemory(StandardSegments))
- DeallocErr = joinErrors(std::move(DeallocErr), errorCodeToError(EC));
+ // Zero out the zero-fill memory.
+ memset(static_cast<char *>(SegMem.base()) + Seg.getContentSize(), 0,
+ Seg.getZeroFillSize());
- DeallocActionsList.pop_back();
- StandardSegmentsList.pop_back();
+ // Record the block for this segment.
+ Blocks[KV.first] = std::move(SegMem);
}
- OnDeallocated(std::move(DeallocErr));
-}
-
-JITLinkMemoryManager::FinalizedAlloc
-InProcessMemoryManager::createFinalizedAlloc(
- sys::MemoryBlock StandardSegments,
- std::vector<AllocActionCall> DeallocActions) {
- std::lock_guard<std::mutex> Lock(FinalizedAllocsMutex);
- auto *FA = FinalizedAllocInfos.Allocate<FinalizedAllocInfo>();
- new (FA) FinalizedAllocInfo(
- {std::move(StandardSegments), std::move(DeallocActions)});
- return FinalizedAlloc(pointerToJITTargetAddress(FA));
+ return std::unique_ptr<InProcessMemoryManager::Allocation>(
+ new IPMMAlloc(std::move(Blocks)));
}
} // end namespace jitlink
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
index a54f2a3..22f17331 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
@@ -107,9 +107,11 @@ MachOLinkGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) {
}
Section &MachOLinkGraphBuilder::getCommonSection() {
- if (!CommonSection)
- CommonSection =
- &G->createSection(CommonSectionName, MemProt::Read | MemProt::Write);
+ if (!CommonSection) {
+ auto Prot = static_cast<sys::Memory::ProtectionFlags>(
+ sys::Memory::MF_READ | sys::Memory::MF_WRITE);
+ CommonSection = &G->createSection(CommonSectionName, Prot);
+ }
return *CommonSection;
}
@@ -174,11 +176,13 @@ Error MachOLinkGraphBuilder::createNormalizedSections() {
// Get prot flags.
// FIXME: Make sure this test is correct (it's probably missing cases
// as-is).
- MemProt Prot;
+ sys::Memory::ProtectionFlags Prot;
if (NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS)
- Prot = MemProt::Read | MemProt::Exec;
+ Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_EXEC);
else
- Prot = MemProt::Read | MemProt::Write;
+ Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_WRITE);
if (!isDebugSection(NSec)) {
auto FullyQualifiedName =
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
index bf81fc4..8a2c7e5 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
@@ -457,14 +457,16 @@ public:
private:
Section &getGOTSection() {
if (!GOTSection)
- GOTSection = &G.createSection("$__GOT", MemProt::Read);
+ GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ);
return *GOTSection;
}
Section &getStubsSection() {
- if (!StubsSection)
- StubsSection =
- &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec);
+ if (!StubsSection) {
+ auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
+ sys::Memory::MF_READ | sys::Memory::MF_EXEC);
+ StubsSection = &G.createSection("$__STUBS", StubsProt);
+ }
return *StubsSection;
}
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
index 40ded07..b860816 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
@@ -479,14 +479,16 @@ public:
private:
Section &getGOTSection() {
if (!GOTSection)
- GOTSection = &G.createSection("$__GOT", MemProt::Read);
+ GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ);
return *GOTSection;
}
Section &getStubsSection() {
- if (!StubsSection)
- StubsSection =
- &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec);
+ if (!StubsSection) {
+ auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
+ sys::Memory::MF_READ | sys::Memory::MF_EXEC);
+ StubsSection = &G.createSection("$__STUBS", StubsProt);
+ }
return *StubsSection;
}
diff --git a/llvm/lib/ExecutionEngine/JITLink/MemoryFlags.cpp b/llvm/lib/ExecutionEngine/JITLink/MemoryFlags.cpp
deleted file mode 100644
index b73a310b..0000000
--- a/llvm/lib/ExecutionEngine/JITLink/MemoryFlags.cpp
+++ /dev/null
@@ -1,33 +0,0 @@
-//===------------- MemoryFlags.cpp - Memory allocation flags --------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ExecutionEngine/JITLink/MemoryFlags.h"
-
-#define DEBUG_TYPE "jitlink"
-
-namespace llvm {
-namespace jitlink {
-
-raw_ostream &operator<<(raw_ostream &OS, MemProt MP) {
- return OS << (((MP & MemProt::Read) != MemProt::None) ? 'R' : '-')
- << (((MP & MemProt::Write) != MemProt::None) ? 'W' : '-')
- << (((MP & MemProt::Exec) != MemProt::None) ? 'X' : '-');
-}
-
-raw_ostream &operator<<(raw_ostream &OS, MemDeallocPolicy MDP) {
- return OS << (MDP == MemDeallocPolicy::Standard ? "standard" : "finalize");
-}
-
-raw_ostream &operator<<(raw_ostream &OS, AllocGroup AG) {
- return OS << '(' << AG.getMemProt() << ", " << AG.getMemDeallocPolicy()
- << ')';
-}
-
-} // end namespace jitlink
-} // end namespace llvm
diff --git a/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp
index fcfe389..36efc744 100644
--- a/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp
@@ -1,15 +1,10 @@
-//===------- DebugObjectManagerPlugin.cpp - JITLink debug objects ---------===//
+//===---- DebugObjectManagerPlugin.h - JITLink debug objects ---*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
-//
-// FIXME: Update Plugin to poke the debug object into a new JITLink section,
-// rather than creating a new allocation.
-//
-//===----------------------------------------------------------------------===//
#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h"
@@ -113,77 +108,70 @@ void ELFDebugObjectSection<ELFT>::dump(raw_ostream &OS, StringRef Name) {
}
}
+static constexpr sys::Memory::ProtectionFlags ReadOnly =
+ static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ);
+
enum class Requirement {
// Request final target memory load-addresses for all sections.
ReportFinalSectionLoadAddresses,
};
-/// The plugin creates a debug object from when JITLink starts processing the
-/// corresponding LinkGraph. It provides access to the pass configuration of
-/// the LinkGraph and calls the finalization function, once the resulting link
-/// artifact was emitted.
+/// The plugin creates a debug object from JITLinkContext when JITLink starts
+/// processing the corresponding LinkGraph. It provides access to the pass
+/// configuration of the LinkGraph and calls the finalization function, once
+/// the resulting link artifact was emitted.
///
class DebugObject {
public:
- DebugObject(JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD,
- ExecutionSession &ES)
- : MemMgr(MemMgr), JD(JD), ES(ES) {}
+ DebugObject(JITLinkContext &Ctx, ExecutionSession &ES) : Ctx(Ctx), ES(ES) {}
void set(Requirement Req) { Reqs.insert(Req); }
bool has(Requirement Req) const { return Reqs.count(Req) > 0; }
- using FinalizeContinuation = std::function<void(Expected<ExecutorAddrRange>)>;
-
+ using FinalizeContinuation = std::function<void(Expected<sys::MemoryBlock>)>;
void finalizeAsync(FinalizeContinuation OnFinalize);
virtual ~DebugObject() {
- if (Alloc) {
- std::vector<FinalizedAlloc> Allocs;
- Allocs.push_back(std::move(Alloc));
- if (Error Err = MemMgr.deallocate(std::move(Allocs)))
+ if (Alloc)
+ if (Error Err = Alloc->deallocate())
ES.reportError(std::move(Err));
- }
}
virtual void reportSectionTargetMemoryRange(StringRef Name,
SectionRange TargetMem) {}
protected:
- using InFlightAlloc = JITLinkMemoryManager::InFlightAlloc;
- using FinalizedAlloc = JITLinkMemoryManager::FinalizedAlloc;
+ using Allocation = JITLinkMemoryManager::Allocation;
- virtual Expected<SimpleSegmentAlloc> finalizeWorkingMemory() = 0;
-
- JITLinkMemoryManager &MemMgr;
- const JITLinkDylib *JD = nullptr;
+ virtual Expected<std::unique_ptr<Allocation>>
+ finalizeWorkingMemory(JITLinkContext &Ctx) = 0;
private:
+ JITLinkContext &Ctx;
ExecutionSession &ES;
std::set<Requirement> Reqs;
- FinalizedAlloc Alloc;
+ std::unique_ptr<Allocation> Alloc{nullptr};
};
// Finalize working memory and take ownership of the resulting allocation. Start
// copying memory over to the target and pass on the result once we're done.
// Ownership of the allocation remains with us for the rest of our lifetime.
void DebugObject::finalizeAsync(FinalizeContinuation OnFinalize) {
- assert(!Alloc && "Cannot finalize more than once");
-
- if (auto SimpleSegAlloc = finalizeWorkingMemory()) {
- auto ROSeg = SimpleSegAlloc->getSegInfo(MemProt::Read);
- ExecutorAddrRange DebugObjRange(ExecutorAddr(ROSeg.Addr),
- ExecutorAddrDiff(ROSeg.WorkingMem.size()));
- SimpleSegAlloc->finalize(
- [this, DebugObjRange,
- OnFinalize = std::move(OnFinalize)](Expected<FinalizedAlloc> FA) {
- if (FA) {
- Alloc = std::move(*FA);
- OnFinalize(DebugObjRange);
- } else
- OnFinalize(FA.takeError());
- });
- } else
- OnFinalize(SimpleSegAlloc.takeError());
+ assert(Alloc == nullptr && "Cannot finalize more than once");
+
+ auto AllocOrErr = finalizeWorkingMemory(Ctx);
+ if (!AllocOrErr)
+ OnFinalize(AllocOrErr.takeError());
+ Alloc = std::move(*AllocOrErr);
+
+ Alloc->finalizeAsync([this, OnFinalize](Error Err) {
+ if (Err)
+ OnFinalize(std::move(Err));
+ else
+ OnFinalize(sys::MemoryBlock(
+ jitTargetAddressToPointer<void *>(Alloc->getTargetMemory(ReadOnly)),
+ Alloc->getWorkingMemory(ReadOnly).size()));
+ });
}
/// The current implementation of ELFDebugObject replicates the approach used in
@@ -202,7 +190,8 @@ public:
StringRef getBuffer() const { return Buffer->getMemBufferRef().getBuffer(); }
protected:
- Expected<SimpleSegmentAlloc> finalizeWorkingMemory() override;
+ Expected<std::unique_ptr<Allocation>>
+ finalizeWorkingMemory(JITLinkContext &Ctx) override;
template <typename ELFT>
Error recordSection(StringRef Name,
@@ -212,16 +201,15 @@ protected:
private:
template <typename ELFT>
static Expected<std::unique_ptr<ELFDebugObject>>
- CreateArchType(MemoryBufferRef Buffer, JITLinkMemoryManager &MemMgr,
- const JITLinkDylib *JD, ExecutionSession &ES);
+ CreateArchType(MemoryBufferRef Buffer, JITLinkContext &Ctx,
+ ExecutionSession &ES);
static std::unique_ptr<WritableMemoryBuffer>
CopyBuffer(MemoryBufferRef Buffer, Error &Err);
ELFDebugObject(std::unique_ptr<WritableMemoryBuffer> Buffer,
- JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD,
- ExecutionSession &ES)
- : DebugObject(MemMgr, JD, ES), Buffer(std::move(Buffer)) {
+ JITLinkContext &Ctx, ExecutionSession &ES)
+ : DebugObject(Ctx, ES), Buffer(std::move(Buffer)) {
set(Requirement::ReportFinalSectionLoadAddresses);
}
@@ -256,14 +244,13 @@ ELFDebugObject::CopyBuffer(MemoryBufferRef Buffer, Error &Err) {
template <typename ELFT>
Expected<std::unique_ptr<ELFDebugObject>>
-ELFDebugObject::CreateArchType(MemoryBufferRef Buffer,
- JITLinkMemoryManager &MemMgr,
- const JITLinkDylib *JD, ExecutionSession &ES) {
+ELFDebugObject::CreateArchType(MemoryBufferRef Buffer, JITLinkContext &Ctx,
+ ExecutionSession &ES) {
using SectionHeader = typename ELFT::Shdr;
Error Err = Error::success();
std::unique_ptr<ELFDebugObject> DebugObj(
- new ELFDebugObject(CopyBuffer(Buffer, Err), MemMgr, JD, ES));
+ new ELFDebugObject(CopyBuffer(Buffer, Err), Ctx, ES));
if (Err)
return std::move(Err);
@@ -312,26 +299,23 @@ ELFDebugObject::Create(MemoryBufferRef Buffer, JITLinkContext &Ctx,
if (Class == ELF::ELFCLASS32) {
if (Endian == ELF::ELFDATA2LSB)
- return CreateArchType<ELF32LE>(Buffer, Ctx.getMemoryManager(),
- Ctx.getJITLinkDylib(), ES);
+ return CreateArchType<ELF32LE>(Buffer, Ctx, ES);
if (Endian == ELF::ELFDATA2MSB)
- return CreateArchType<ELF32BE>(Buffer, Ctx.getMemoryManager(),
- Ctx.getJITLinkDylib(), ES);
+ return CreateArchType<ELF32BE>(Buffer, Ctx, ES);
return nullptr;
}
if (Class == ELF::ELFCLASS64) {
if (Endian == ELF::ELFDATA2LSB)
- return CreateArchType<ELF64LE>(Buffer, Ctx.getMemoryManager(),
- Ctx.getJITLinkDylib(), ES);
+ return CreateArchType<ELF64LE>(Buffer, Ctx, ES);
if (Endian == ELF::ELFDATA2MSB)
- return CreateArchType<ELF64BE>(Buffer, Ctx.getMemoryManager(),
- Ctx.getJITLinkDylib(), ES);
+ return CreateArchType<ELF64BE>(Buffer, Ctx, ES);
return nullptr;
}
return nullptr;
}
-Expected<SimpleSegmentAlloc> ELFDebugObject::finalizeWorkingMemory() {
+Expected<std::unique_ptr<DebugObject::Allocation>>
+ELFDebugObject::finalizeWorkingMemory(JITLinkContext &Ctx) {
LLVM_DEBUG({
dbgs() << "Section load-addresses in debug object for \""
<< Buffer->getBufferIdentifier() << "\":\n";
@@ -340,21 +324,28 @@ Expected<SimpleSegmentAlloc> ELFDebugObject::finalizeWorkingMemory() {
});
// TODO: This works, but what actual alignment requirements do we have?
- unsigned PageSize = sys::Process::getPageSizeEstimate();
+ unsigned Alignment = sys::Process::getPageSizeEstimate();
+ JITLinkMemoryManager &MemMgr = Ctx.getMemoryManager();
+ const JITLinkDylib *JD = Ctx.getJITLinkDylib();
size_t Size = Buffer->getBufferSize();
// Allocate working memory for debug object in read-only segment.
- auto Alloc = SimpleSegmentAlloc::Create(
- MemMgr, JD, {{MemProt::Read, {Size, Align(PageSize)}}});
- if (!Alloc)
- return Alloc;
+ JITLinkMemoryManager::SegmentsRequestMap SingleReadOnlySegment;
+ SingleReadOnlySegment[ReadOnly] =
+ JITLinkMemoryManager::SegmentRequest(Alignment, Size, 0);
+
+ auto AllocOrErr = MemMgr.allocate(JD, SingleReadOnlySegment);
+ if (!AllocOrErr)
+ return AllocOrErr.takeError();
// Initialize working memory with a copy of our object buffer.
- auto SegInfo = Alloc->getSegInfo(MemProt::Read);
- memcpy(SegInfo.WorkingMem.data(), Buffer->getBufferStart(), Size);
+ // TODO: Use our buffer as working memory directly.
+ std::unique_ptr<Allocation> Alloc = std::move(*AllocOrErr);
+ MutableArrayRef<char> WorkingMem = Alloc->getWorkingMemory(ReadOnly);
+ memcpy(WorkingMem.data(), Buffer->getBufferStart(), Size);
Buffer.reset();
- return Alloc;
+ return std::move(Alloc);
}
void ELFDebugObject::reportSectionTargetMemoryRange(StringRef Name,
@@ -456,7 +447,7 @@ Error DebugObjectManagerPlugin::notifyEmitted(
std::future<MSVCPError> FinalizeErr = FinalizePromise.get_future();
It->second->finalizeAsync(
- [this, &FinalizePromise, &MR](Expected<ExecutorAddrRange> TargetMem) {
+ [this, &FinalizePromise, &MR](Expected<sys::MemoryBlock> TargetMem) {
// Any failure here will fail materialization.
if (!TargetMem) {
FinalizePromise.set_value(TargetMem.takeError());
diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
index 88d4cf9..79c602d 100644
--- a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
@@ -56,7 +56,7 @@ public:
"<DSOHandleMU>", TT, PointerSize, Endianness,
jitlink::getGenericEdgeKindName);
auto &DSOHandleSection =
- G->createSection(".data.__dso_handle", jitlink::MemProt::Read);
+ G->createSection(".data.__dso_handle", sys::Memory::MF_READ);
auto &DSOHandleBlock = G->createContentBlock(
DSOHandleSection, getDSOHandleContent(PointerSize), 0, 8, 0);
auto &DSOHandleSymbol = G->addDefinedSymbol(
diff --git a/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp b/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp
index f3fe055..e1639a8 100644
--- a/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp
@@ -43,9 +43,10 @@ createJITLoaderGDBRegistrar(ExecutionSession &ES) {
ES, ExecutorAddr((*Result)[0][0]));
}
-Error EPCDebugObjectRegistrar::registerDebugObject(
- ExecutorAddrRange TargetMem) {
- return ES.callSPSWrapper<void(SPSExecutorAddrRange)>(RegisterFn, TargetMem);
+Error EPCDebugObjectRegistrar::registerDebugObject(sys::MemoryBlock TargetMem) {
+ return ES.callSPSWrapper<void(SPSExecutorAddr, uint64_t)>(
+ RegisterFn, ExecutorAddr::fromPtr(TargetMem.base()),
+ static_cast<uint64_t>(TargetMem.allocatedSize()));
}
} // namespace orc
diff --git a/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp
index 9456837..234c085 100644
--- a/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp
@@ -7,168 +7,132 @@
//===----------------------------------------------------------------------===//
#include "llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h"
-
-#include "llvm/ExecutionEngine/JITLink/JITLink.h"
#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
#include <limits>
-using namespace llvm::jitlink;
-
namespace llvm {
namespace orc {
-class EPCGenericJITLinkMemoryManager::InFlightAlloc
- : public jitlink::JITLinkMemoryManager::InFlightAlloc {
+class EPCGenericJITLinkMemoryManager::Alloc
+ : public jitlink::JITLinkMemoryManager::Allocation {
public:
struct SegInfo {
char *WorkingMem = nullptr;
- ExecutorAddr Addr;
+ ExecutorAddr TargetAddr;
uint64_t ContentSize = 0;
uint64_t ZeroFillSize = 0;
};
- using SegInfoMap = AllocGroupSmallMap<SegInfo>;
+ using SegInfoMap = DenseMap<unsigned, SegInfo>;
+
+ Alloc(EPCGenericJITLinkMemoryManager &Parent, ExecutorAddr TargetAddr,
+ std::unique_ptr<char[]> WorkingBuffer, SegInfoMap Segs)
+ : Parent(Parent), TargetAddr(TargetAddr),
+ WorkingBuffer(std::move(WorkingBuffer)), Segs(std::move(Segs)) {}
+
+ MutableArrayRef<char> getWorkingMemory(ProtectionFlags Seg) override {
+ auto I = Segs.find(Seg);
+ assert(I != Segs.end() && "No allocation for seg");
+ assert(I->second.ContentSize <= std::numeric_limits<size_t>::max());
+ return {I->second.WorkingMem, static_cast<size_t>(I->second.ContentSize)};
+ }
- InFlightAlloc(EPCGenericJITLinkMemoryManager &Parent, LinkGraph &G,
- ExecutorAddr AllocAddr, SegInfoMap Segs)
- : Parent(Parent), G(G), AllocAddr(AllocAddr), Segs(std::move(Segs)) {}
+ JITTargetAddress getTargetMemory(ProtectionFlags Seg) override {
+ auto I = Segs.find(Seg);
+ assert(I != Segs.end() && "No allocation for seg");
+ return I->second.TargetAddr.getValue();
+ }
- void finalize(OnFinalizedFunction OnFinalize) override {
+ void finalizeAsync(FinalizeContinuation OnFinalize) override {
+ char *WorkingMem = WorkingBuffer.get();
tpctypes::FinalizeRequest FR;
for (auto &KV : Segs) {
assert(KV.second.ContentSize <= std::numeric_limits<size_t>::max());
FR.Segments.push_back(tpctypes::SegFinalizeRequest{
tpctypes::toWireProtectionFlags(
- toSysMemoryProtectionFlags(KV.first.getMemProt())),
- KV.second.Addr,
+ static_cast<sys::Memory::ProtectionFlags>(KV.first)),
+ KV.second.TargetAddr,
alignTo(KV.second.ContentSize + KV.second.ZeroFillSize,
Parent.EPC.getPageSize()),
- {KV.second.WorkingMem, static_cast<size_t>(KV.second.ContentSize)}});
+ {WorkingMem, static_cast<size_t>(KV.second.ContentSize)}});
+ WorkingMem += KV.second.ContentSize;
}
-
- // Transfer allocation actions.
- // FIXME: Merge JITLink and ORC SupportFunctionCall and Action list types,
- // turn this into a std::swap.
- FR.Actions.reserve(G.allocActions().size());
- for (auto &ActPair : G.allocActions())
- FR.Actions.push_back(
- {{ExecutorAddr(ActPair.Finalize.FnAddr),
- ExecutorAddr(ActPair.Finalize.CtxAddr), ActPair.Finalize.CtxSize},
- {ExecutorAddr(ActPair.Dealloc.FnAddr),
- ExecutorAddr(ActPair.Dealloc.CtxAddr), ActPair.Dealloc.CtxSize}});
- G.allocActions().clear();
-
Parent.EPC.callSPSWrapperAsync<
rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>(
Parent.SAs.Finalize,
- [OnFinalize = std::move(OnFinalize), AllocAddr = this->AllocAddr](
- Error SerializationErr, Error FinalizeErr) mutable {
- // FIXME: Release abandoned alloc.
+ [OnFinalize = std::move(OnFinalize)](Error SerializationErr,
+ Error FinalizeErr) {
if (SerializationErr) {
cantFail(std::move(FinalizeErr));
OnFinalize(std::move(SerializationErr));
- } else if (FinalizeErr)
+ } else
OnFinalize(std::move(FinalizeErr));
- else
- OnFinalize(FinalizedAlloc(AllocAddr.getValue()));
},
Parent.SAs.Allocator, std::move(FR));
}
- void abandon(OnAbandonedFunction OnAbandoned) override {
- // FIXME: Return memory to pool instead.
- Parent.EPC.callSPSWrapperAsync<
- rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>(
- Parent.SAs.Deallocate,
- [OnAbandoned = std::move(OnAbandoned)](Error SerializationErr,
- Error DeallocateErr) mutable {
- if (SerializationErr) {
- cantFail(std::move(DeallocateErr));
- OnAbandoned(std::move(SerializationErr));
- } else
- OnAbandoned(std::move(DeallocateErr));
- },
- Parent.SAs.Allocator, ArrayRef<ExecutorAddr>(AllocAddr));
+ Error deallocate() override {
+ Error Err = Error::success();
+ if (auto E2 = Parent.EPC.callSPSWrapper<
+ rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>(
+ Parent.SAs.Deallocate, Err, Parent.SAs.Allocator,
+ ArrayRef<ExecutorAddr>(TargetAddr)))
+ return E2;
+ return Err;
}
private:
EPCGenericJITLinkMemoryManager &Parent;
- LinkGraph &G;
- ExecutorAddr AllocAddr;
+ ExecutorAddr TargetAddr;
+ std::unique_ptr<char[]> WorkingBuffer;
SegInfoMap Segs;
};
-void EPCGenericJITLinkMemoryManager::allocate(const JITLinkDylib *JD,
- LinkGraph &G,
- OnAllocatedFunction OnAllocated) {
- BasicLayout BL(G);
-
- auto Pages = BL.getContiguousPageBasedLayoutSizes(EPC.getPageSize());
- if (!Pages)
- return OnAllocated(Pages.takeError());
-
- EPC.callSPSWrapperAsync<rt::SPSSimpleExecutorMemoryManagerReserveSignature>(
- SAs.Reserve,
- [this, BL = std::move(BL), OnAllocated = std::move(OnAllocated)](
- Error SerializationErr, Expected<ExecutorAddr> AllocAddr) mutable {
- if (SerializationErr) {
- cantFail(AllocAddr.takeError());
- return OnAllocated(std::move(SerializationErr));
- }
- if (!AllocAddr)
- return OnAllocated(AllocAddr.takeError());
-
- completeAllocation(*AllocAddr, std::move(BL), std::move(OnAllocated));
- },
- SAs.Allocator, Pages->total());
-}
-
-void EPCGenericJITLinkMemoryManager::deallocate(
- std::vector<FinalizedAlloc> Allocs, OnDeallocatedFunction OnDeallocated) {
- EPC.callSPSWrapperAsync<
- rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>(
- SAs.Deallocate,
- [OnDeallocated = std::move(OnDeallocated)](Error SerErr,
- Error DeallocErr) mutable {
- if (SerErr) {
- cantFail(std::move(DeallocErr));
- OnDeallocated(std::move(SerErr));
- } else
- OnDeallocated(std::move(DeallocErr));
- },
- SAs.Allocator, Allocs);
- for (auto &A : Allocs)
- A.release();
-}
-
-void EPCGenericJITLinkMemoryManager::completeAllocation(
- ExecutorAddr AllocAddr, BasicLayout BL, OnAllocatedFunction OnAllocated) {
-
- InFlightAlloc::SegInfoMap SegInfos;
+Expected<std::unique_ptr<jitlink::JITLinkMemoryManager::Allocation>>
+EPCGenericJITLinkMemoryManager::allocate(const jitlink::JITLinkDylib *JD,
+ const SegmentsRequestMap &Request) {
+ Alloc::SegInfoMap Segs;
+ uint64_t AllocSize = 0;
+ size_t WorkingSize = 0;
+ for (auto &KV : Request) {
+ if (!isPowerOf2_64(KV.second.getAlignment()))
+ return make_error<StringError>("Alignment is not a power of two",
+ inconvertibleErrorCode());
+ if (KV.second.getAlignment() > EPC.getPageSize())
+ return make_error<StringError>("Alignment exceeds page size",
+ inconvertibleErrorCode());
+
+ auto &Seg = Segs[KV.first];
+ Seg.ContentSize = KV.second.getContentSize();
+ Seg.ZeroFillSize = KV.second.getZeroFillSize();
+ AllocSize += alignTo(Seg.ContentSize + Seg.ZeroFillSize, EPC.getPageSize());
+ WorkingSize += Seg.ContentSize;
+ }
- ExecutorAddr NextSegAddr = AllocAddr;
- for (auto &KV : BL.segments()) {
- const auto &AG = KV.first;
+ std::unique_ptr<char[]> WorkingBuffer;
+ if (WorkingSize > 0)
+ WorkingBuffer = std::make_unique<char[]>(WorkingSize);
+ Expected<ExecutorAddr> TargetAllocAddr((ExecutorAddr()));
+ if (auto Err = EPC.callSPSWrapper<
+ rt::SPSSimpleExecutorMemoryManagerReserveSignature>(
+ SAs.Reserve, TargetAllocAddr, SAs.Allocator, AllocSize))
+ return std::move(Err);
+ if (!TargetAllocAddr)
+ return TargetAllocAddr.takeError();
+
+ char *WorkingMem = WorkingBuffer.get();
+ JITTargetAddress SegAddr = TargetAllocAddr->getValue();
+ for (auto &KV : Segs) {
auto &Seg = KV.second;
-
- Seg.Addr = NextSegAddr.getValue();
- KV.second.WorkingMem = BL.getGraph().allocateBuffer(Seg.ContentSize).data();
- NextSegAddr += ExecutorAddrDiff(
- alignTo(Seg.ContentSize + Seg.ZeroFillSize, EPC.getPageSize()));
-
- auto &SegInfo = SegInfos[AG];
- SegInfo.ContentSize = Seg.ContentSize;
- SegInfo.ZeroFillSize = Seg.ZeroFillSize;
- SegInfo.Addr = ExecutorAddr(Seg.Addr);
- SegInfo.WorkingMem = Seg.WorkingMem;
+ Seg.TargetAddr.setValue(SegAddr);
+ SegAddr += alignTo(Seg.ContentSize + Seg.ZeroFillSize, EPC.getPageSize());
+ Seg.WorkingMem = WorkingMem;
+ WorkingMem += Seg.ContentSize;
}
- if (auto Err = BL.apply())
- return OnAllocated(std::move(Err));
-
- OnAllocated(std::make_unique<InFlightAlloc>(*this, BL.getGraph(), AllocAddr,
- std::move(SegInfos)));
+ return std::make_unique<Alloc>(*this, *TargetAllocAddr,
+ std::move(WorkingBuffer), std::move(Segs));
}
} // end namespace orc
diff --git a/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp
index 948d1d9..b9c70b0 100644
--- a/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp
@@ -43,12 +43,12 @@ public:
protected:
Error grow() override;
- using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc;
+ using Allocation = jitlink::JITLinkMemoryManager::Allocation;
EPCIndirectionUtils &EPCIU;
unsigned TrampolineSize = 0;
unsigned TrampolinesPerPage = 0;
- std::vector<FinalizedAlloc> TrampolineBlocks;
+ std::vector<std::unique_ptr<Allocation>> TrampolineBlocks;
};
class EPCIndirectStubsManager : public IndirectStubsManager,
@@ -89,19 +89,12 @@ EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
Error EPCTrampolinePool::deallocatePool() {
Error Err = Error::success();
- std::promise<MSVCPError> DeallocResultP;
- auto DeallocResultF = DeallocResultP.get_future();
-
- EPCIU.getExecutorProcessControl().getMemMgr().deallocate(
- std::move(TrampolineBlocks),
- [&](Error Err) { DeallocResultP.set_value(std::move(Err)); });
-
- return DeallocResultF.get();
+ for (auto &Alloc : TrampolineBlocks)
+ Err = joinErrors(std::move(Err), Alloc->deallocate());
+ return Err;
}
Error EPCTrampolinePool::grow() {
- using namespace jitlink;
-
assert(AvailableTrampolines.empty() &&
"Grow called with trampolines still available");
@@ -109,26 +102,34 @@ Error EPCTrampolinePool::grow() {
assert(ResolverAddress && "Resolver address can not be null");
auto &EPC = EPCIU.getExecutorProcessControl();
+ constexpr auto TrampolinePagePermissions =
+ static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_EXEC);
auto PageSize = EPC.getPageSize();
- auto Alloc = SimpleSegmentAlloc::Create(
- EPC.getMemMgr(), nullptr,
- {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
+ jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
+ Request[TrampolinePagePermissions] = {PageSize, static_cast<size_t>(PageSize),
+ 0};
+ auto Alloc = EPC.getMemMgr().allocate(nullptr, Request);
+
if (!Alloc)
return Alloc.takeError();
unsigned NumTrampolines = TrampolinesPerPage;
- auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
- EPCIU.getABISupport().writeTrampolines(
- SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines);
+ auto WorkingMemory = (*Alloc)->getWorkingMemory(TrampolinePagePermissions);
+ auto TargetAddress = (*Alloc)->getTargetMemory(TrampolinePagePermissions);
+
+ EPCIU.getABISupport().writeTrampolines(WorkingMemory.data(), TargetAddress,
+ ResolverAddress, NumTrampolines);
+
+ auto TargetAddr = (*Alloc)->getTargetMemory(TrampolinePagePermissions);
for (unsigned I = 0; I < NumTrampolines; ++I)
- AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize));
+ AvailableTrampolines.push_back(TargetAddr + (I * TrampolineSize));
- auto FA = Alloc->finalize();
- if (!FA)
- return FA.takeError();
+ if (auto Err = (*Alloc)->finalize())
+ return Err;
- TrampolineBlocks.push_back(std::move(*FA));
+ TrampolineBlocks.push_back(std::move(*Alloc));
return Error::success();
}
@@ -266,17 +267,17 @@ EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {
}
Error EPCIndirectionUtils::cleanup() {
+ Error Err = Error::success();
- auto &MemMgr = EPC.getMemMgr();
- auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs));
+ for (auto &A : IndirectStubAllocs)
+ Err = joinErrors(std::move(Err), A->deallocate());
if (TP)
Err = joinErrors(std::move(Err),
static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
if (ResolverBlock)
- Err =
- joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock)));
+ Err = joinErrors(std::move(Err), ResolverBlock->deallocate());
return Err;
}
@@ -284,29 +285,29 @@ Error EPCIndirectionUtils::cleanup() {
Expected<JITTargetAddress>
EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr,
JITTargetAddress ReentryCtxAddr) {
- using namespace jitlink;
-
assert(ABI && "ABI can not be null");
+ constexpr auto ResolverBlockPermissions =
+ static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_EXEC);
auto ResolverSize = ABI->getResolverCodeSize();
- auto Alloc =
- SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr,
- {{MemProt::Read | MemProt::Exec,
- {ResolverSize, Align(EPC.getPageSize())}}});
-
+ jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
+ Request[ResolverBlockPermissions] = {EPC.getPageSize(),
+ static_cast<size_t>(ResolverSize), 0};
+ auto Alloc = EPC.getMemMgr().allocate(nullptr, Request);
if (!Alloc)
return Alloc.takeError();
- auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
- ABI->writeResolverCode(SegInfo.WorkingMem.data(), SegInfo.Addr, ReentryFnAddr,
+ auto WorkingMemory = (*Alloc)->getWorkingMemory(ResolverBlockPermissions);
+ ResolverBlockAddr = (*Alloc)->getTargetMemory(ResolverBlockPermissions);
+ ABI->writeResolverCode(WorkingMemory.data(), ResolverBlockAddr, ReentryFnAddr,
ReentryCtxAddr);
- auto FA = Alloc->finalize();
- if (!FA)
- return FA.takeError();
+ if (auto Err = (*Alloc)->finalize())
+ return std::move(Err);
- ResolverBlock = std::move(*FA);
- return SegInfo.Addr;
+ ResolverBlock = std::move(*Alloc);
+ return ResolverBlockAddr;
}
std::unique_ptr<IndirectStubsManager>
@@ -340,7 +341,6 @@ EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
Expected<EPCIndirectionUtils::IndirectStubInfoVector>
EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
- using namespace jitlink;
std::lock_guard<std::mutex> Lock(EPCUIMutex);
@@ -350,40 +350,42 @@ EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
auto PageSize = EPC.getPageSize();
auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
NumStubsToAllocate = StubBytes / ABI->getStubSize();
- auto PtrBytes =
+ auto PointerBytes =
alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
- auto StubProt = MemProt::Read | MemProt::Exec;
- auto PtrProt = MemProt::Read | MemProt::Write;
-
- auto Alloc = SimpleSegmentAlloc::Create(
- EPC.getMemMgr(), nullptr,
- {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
- {PtrProt, {PtrBytes, Align(PageSize)}}});
-
+ constexpr auto StubPagePermissions =
+ static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_EXEC);
+ constexpr auto PointerPagePermissions =
+ static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_WRITE);
+
+ jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
+ Request[StubPagePermissions] = {PageSize, static_cast<size_t>(StubBytes),
+ 0};
+ Request[PointerPagePermissions] = {PageSize, 0, PointerBytes};
+ auto Alloc = EPC.getMemMgr().allocate(nullptr, Request);
if (!Alloc)
return Alloc.takeError();
- auto StubSeg = Alloc->getSegInfo(StubProt);
- auto PtrSeg = Alloc->getSegInfo(PtrProt);
-
- ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr,
- PtrSeg.Addr, NumStubsToAllocate);
+ auto StubTargetAddr = (*Alloc)->getTargetMemory(StubPagePermissions);
+ auto PointerTargetAddr = (*Alloc)->getTargetMemory(PointerPagePermissions);
- auto FA = Alloc->finalize();
- if (!FA)
- return FA.takeError();
+ ABI->writeIndirectStubsBlock(
+ (*Alloc)->getWorkingMemory(StubPagePermissions).data(), StubTargetAddr,
+ PointerTargetAddr, NumStubsToAllocate);
- IndirectStubAllocs.push_back(std::move(*FA));
+ if (auto Err = (*Alloc)->finalize())
+ return std::move(Err);
- auto StubExecutorAddr = StubSeg.Addr;
- auto PtrExecutorAddr = PtrSeg.Addr;
for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
AvailableIndirectStubs.push_back(
- IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr));
- StubExecutorAddr += ABI->getStubSize();
- PtrExecutorAddr += ABI->getPointerSize();
+ IndirectStubInfo(StubTargetAddr, PointerTargetAddr));
+ StubTargetAddr += ABI->getStubSize();
+ PointerTargetAddr += ABI->getPointerSize();
}
+
+ IndirectStubAllocs.push_back(std::move(*Alloc));
}
assert(NumStubs <= AvailableIndirectStubs.size() &&
diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp
index 0145387..6fb8b52 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp
@@ -31,8 +31,7 @@ SelfExecutorProcessControl::SelfExecutorProcessControl(
OwnedMemMgr = std::move(MemMgr);
if (!OwnedMemMgr)
- OwnedMemMgr = std::make_unique<jitlink::InProcessMemoryManager>(
- sys::Process::getPageSizeEstimate());
+ OwnedMemMgr = std::make_unique<jitlink::InProcessMemoryManager>();
this->TargetTriple = std::move(TargetTriple);
this->PageSize = PageSize;
diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
index e346262..1705010 100644
--- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
@@ -53,7 +53,7 @@ public:
auto G = std::make_unique<jitlink::LinkGraph>(
"<MachOHeaderMU>", TT, PointerSize, Endianness,
jitlink::getGenericEdgeKindName);
- auto &HeaderSection = G->createSection("__header", jitlink::MemProt::Read);
+ auto &HeaderSection = G->createSection("__header", sys::Memory::MF_READ);
auto &HeaderBlock = createHeaderBlock(*G, HeaderSection);
// Init symbol is header-start symbol.
diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
index 22a6425..40d4f19 100644
--- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
@@ -306,7 +306,8 @@ public:
return Error::success();
}
- void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A) override {
+ void notifyFinalized(
+ std::unique_ptr<JITLinkMemoryManager::Allocation> A) override {
if (auto Err = Layer.notifyEmitted(*MR, std::move(A))) {
Layer.getExecutionSession().reportError(std::move(Err));
MR->failMaterialization();
@@ -679,7 +680,7 @@ void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) {
}
Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR,
- FinalizedAlloc FA) {
+ AllocPtr Alloc) {
Error Err = Error::success();
for (auto &P : Plugins)
Err = joinErrors(std::move(Err), P->notifyEmitted(MR));
@@ -688,20 +689,17 @@ Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR,
return Err;
return MR.withResourceKeyDo(
- [&](ResourceKey K) { Allocs[K].push_back(std::move(FA)); });
+ [&](ResourceKey K) { Allocs[K].push_back(std::move(Alloc)); });
}
Error ObjectLinkingLayer::handleRemoveResources(ResourceKey K) {
- {
- Error Err = Error::success();
- for (auto &P : Plugins)
- Err = joinErrors(std::move(Err), P->notifyRemovingResources(K));
- if (Err)
- return Err;
- }
+ Error Err = Error::success();
- std::vector<FinalizedAlloc> AllocsToRemove;
+ for (auto &P : Plugins)
+ Err = joinErrors(std::move(Err), P->notifyRemovingResources(K));
+
+ std::vector<AllocPtr> AllocsToRemove;
getExecutionSession().runSessionLocked([&] {
auto I = Allocs.find(K);
if (I != Allocs.end()) {
@@ -710,7 +708,12 @@ Error ObjectLinkingLayer::handleRemoveResources(ResourceKey K) {
}
});
- return MemMgr.deallocate(std::move(AllocsToRemove));
+ while (!AllocsToRemove.empty()) {
+ Err = joinErrors(std::move(Err), AllocsToRemove.back()->deallocate());
+ AllocsToRemove.pop_back();
+ }
+
+ return Err;
}
void ObjectLinkingLayer::handleTransferResources(ResourceKey DstKey,
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp
index 787a6c6..0ee194c 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp
@@ -64,16 +64,15 @@ LLVM_ATTRIBUTE_NOINLINE void __jit_debug_register_code() {
}
using namespace llvm;
-using namespace llvm::orc;
// Serialize rendezvous with the debugger as well as access to shared data.
ManagedStatic<std::mutex> JITDebugLock;
// Register debug object, return error message or null for success.
-static void registerJITLoaderGDBImpl(ExecutorAddrRange DebugObjRange) {
+static void registerJITLoaderGDBImpl(JITTargetAddress Addr, uint64_t Size) {
jit_code_entry *E = new jit_code_entry;
- E->symfile_addr = DebugObjRange.Start.toPtr<const char *>();
- E->symfile_size = DebugObjRange.size().getValue();
+ E->symfile_addr = jitTargetAddressToPointer<const char *>(Addr);
+ E->symfile_size = Size;
E->prev_entry = nullptr;
std::lock_guard<std::mutex> Lock(*JITDebugLock);
@@ -96,7 +95,7 @@ static void registerJITLoaderGDBImpl(ExecutorAddrRange DebugObjRange) {
extern "C" orc::shared::detail::CWrapperFunctionResult
llvm_orc_registerJITLoaderGDBWrapper(const char *Data, uint64_t Size) {
using namespace orc::shared;
- return WrapperFunction<void(SPSExecutorAddrRange)>::handle(
+ return WrapperFunction<void(SPSExecutorAddr, uint64_t)>::handle(
Data, Size, registerJITLoaderGDBImpl)
.release();
}