diff options
Diffstat (limited to 'llvm/lib')
-rw-r--r-- | llvm/lib/LTO/LTO.cpp | 10 | ||||
-rw-r--r-- | llvm/lib/LTO/LTOCodeGenerator.cpp | 8 | ||||
-rw-r--r-- | llvm/lib/LTO/ThinLTOCodeGenerator.cpp | 6 | ||||
-rw-r--r-- | llvm/lib/Transforms/IPO/LowerTypeTests.cpp | 60 | ||||
-rw-r--r-- | llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp | 79 |
5 files changed, 149 insertions, 14 deletions
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 297b11d..e8f0fd6 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -982,6 +982,11 @@ Error LTO::runRegularLTO(AddStreamFn AddStream) { } } + // If allowed, upgrade public vcall visibility metadata to linkage unit + // visibility before whole program devirtualization in the optimizer. + updateVCallVisibilityInModule(*RegularLTO.CombinedModule, + Conf.HasWholeProgramVisibility); + if (Conf.PreOptModuleHook && !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule)) return Error::success(); @@ -1299,6 +1304,11 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache, std::set<GlobalValue::GUID> ExportedGUIDs; + // If allowed, upgrade public vcall visibility to linkage unit visibility in + // the summaries before whole program devirtualization below. + updateVCallVisibilityInIndex(ThinLTO.CombinedIndex, + Conf.HasWholeProgramVisibility); + // Perform index-based WPD. This will return immediately if there are // no index entries in the typeIdMetadata map (e.g. if we are instead // performing IR-based WPD in hybrid regular/thin LTO mode). diff --git a/llvm/lib/LTO/LTOCodeGenerator.cpp b/llvm/lib/LTO/LTOCodeGenerator.cpp index 5fef142..b3bc727 100644 --- a/llvm/lib/LTO/LTOCodeGenerator.cpp +++ b/llvm/lib/LTO/LTOCodeGenerator.cpp @@ -57,6 +57,7 @@ #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/Internalize.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#include "llvm/Transforms/IPO/WholeProgramDevirt.h" #include "llvm/Transforms/ObjCARC.h" #include "llvm/Transforms/Utils/ModuleUtils.h" #include <system_error> @@ -542,6 +543,13 @@ bool LTOCodeGenerator::optimize(bool DisableVerify, bool DisableInline, } StatsFile = std::move(StatsFileOrErr.get()); + // Currently there is no support for enabling whole program visibility via a + // linker option in the old LTO API, but this call allows it to be specified + // via the internal option. Must be done before WPD invoked via the optimizer + // pipeline run below. + updateVCallVisibilityInModule(*MergedModule, + /* WholeProgramVisibilityEnabledInLTO */ false); + // We always run the verifier once on the merged module, the `DisableVerify` // parameter only applies to subsequent verify. verifyMergedModuleOnce(); diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp index 0bb518b..f4099e6 100644 --- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp +++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp @@ -969,6 +969,12 @@ void ThinLTOCodeGenerator::run() { // Synthesize entry counts for functions in the combined index. computeSyntheticCounts(*Index); + // Currently there is no support for enabling whole program visibility via a + // linker option in the old LTO API, but this call allows it to be specified + // via the internal option. Must be done before WPD below. + updateVCallVisibilityInIndex(*Index, + /* WholeProgramVisibilityEnabledInLTO */ false); + // Perform index-based WPD. This will return immediately if there are // no index entries in the typeIdMetadata map (e.g. if we are instead // performing IR-based WPD in hybrid regular/thin LTO mode). diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp index e6747a6..6eba35a 100644 --- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp +++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp @@ -382,6 +382,9 @@ class LowerTypeTestsModule { ModuleSummaryIndex *ExportSummary; const ModuleSummaryIndex *ImportSummary; + // Set when the client has invoked this to simply drop all type test assume + // sequences. + bool DropTypeTests; Triple::ArchType Arch; Triple::OSType OS; @@ -500,7 +503,8 @@ class LowerTypeTestsModule { public: LowerTypeTestsModule(Module &M, ModuleSummaryIndex *ExportSummary, - const ModuleSummaryIndex *ImportSummary); + const ModuleSummaryIndex *ImportSummary, + bool DropTypeTests); bool lower(); @@ -516,22 +520,24 @@ struct LowerTypeTests : public ModulePass { ModuleSummaryIndex *ExportSummary; const ModuleSummaryIndex *ImportSummary; + bool DropTypeTests; LowerTypeTests() : ModulePass(ID), UseCommandLine(true) { initializeLowerTypeTestsPass(*PassRegistry::getPassRegistry()); } LowerTypeTests(ModuleSummaryIndex *ExportSummary, - const ModuleSummaryIndex *ImportSummary) + const ModuleSummaryIndex *ImportSummary, bool DropTypeTests) : ModulePass(ID), ExportSummary(ExportSummary), - ImportSummary(ImportSummary) { + ImportSummary(ImportSummary), DropTypeTests(DropTypeTests) { initializeLowerTypeTestsPass(*PassRegistry::getPassRegistry()); } bool runOnModule(Module &M) override { if (UseCommandLine) return LowerTypeTestsModule::runForTesting(M); - return LowerTypeTestsModule(M, ExportSummary, ImportSummary).lower(); + return LowerTypeTestsModule(M, ExportSummary, ImportSummary, DropTypeTests) + .lower(); } }; @@ -544,8 +550,9 @@ INITIALIZE_PASS(LowerTypeTests, "lowertypetests", "Lower type metadata", false, ModulePass * llvm::createLowerTypeTestsPass(ModuleSummaryIndex *ExportSummary, - const ModuleSummaryIndex *ImportSummary) { - return new LowerTypeTests(ExportSummary, ImportSummary); + const ModuleSummaryIndex *ImportSummary, + bool DropTypeTests) { + return new LowerTypeTests(ExportSummary, ImportSummary, DropTypeTests); } /// Build a bit set for TypeId using the object layouts in @@ -1655,8 +1662,9 @@ void LowerTypeTestsModule::buildBitSetsFromDisjointSet( /// Lower all type tests in this module. LowerTypeTestsModule::LowerTypeTestsModule( Module &M, ModuleSummaryIndex *ExportSummary, - const ModuleSummaryIndex *ImportSummary) - : M(M), ExportSummary(ExportSummary), ImportSummary(ImportSummary) { + const ModuleSummaryIndex *ImportSummary, bool DropTypeTests) + : M(M), ExportSummary(ExportSummary), ImportSummary(ImportSummary), + DropTypeTests(DropTypeTests) { assert(!(ExportSummary && ImportSummary)); Triple TargetTriple(M.getTargetTriple()); Arch = TargetTriple.getArch(); @@ -1683,7 +1691,8 @@ bool LowerTypeTestsModule::runForTesting(Module &M) { bool Changed = LowerTypeTestsModule( M, ClSummaryAction == PassSummaryAction::Export ? &Summary : nullptr, - ClSummaryAction == PassSummaryAction::Import ? &Summary : nullptr) + ClSummaryAction == PassSummaryAction::Import ? &Summary : nullptr, + /*DropTypeTests*/ false) .lower(); if (!ClWriteSummary.empty()) { @@ -1750,6 +1759,33 @@ void LowerTypeTestsModule::replaceDirectCalls(Value *Old, Value *New) { } bool LowerTypeTestsModule::lower() { + Function *TypeTestFunc = + M.getFunction(Intrinsic::getName(Intrinsic::type_test)); + + if (DropTypeTests && TypeTestFunc) { + for (auto UI = TypeTestFunc->use_begin(), UE = TypeTestFunc->use_end(); + UI != UE;) { + auto *CI = cast<CallInst>((*UI++).getUser()); + // Find and erase llvm.assume intrinsics for this llvm.type.test call. + for (auto CIU = CI->use_begin(), CIUE = CI->use_end(); CIU != CIUE;) { + if (auto *AssumeCI = dyn_cast<CallInst>((*CIU++).getUser())) { + Function *F = AssumeCI->getCalledFunction(); + if (F && F->getIntrinsicID() == Intrinsic::assume) + AssumeCI->eraseFromParent(); + } + } + CI->eraseFromParent(); + } + + // We have deleted the type intrinsics, so we no longer have enough + // information to reason about the liveness of virtual function pointers + // in GlobalDCE. + for (GlobalVariable &GV : M.globals()) + GV.eraseMetadata(LLVMContext::MD_vcall_visibility); + + return true; + } + // If only some of the modules were split, we cannot correctly perform // this transformation. We already checked for the presense of type tests // with partially split modules during the thin link, and would have emitted @@ -1758,8 +1794,6 @@ bool LowerTypeTestsModule::lower() { (ImportSummary && ImportSummary->partiallySplitLTOUnits())) return false; - Function *TypeTestFunc = - M.getFunction(Intrinsic::getName(Intrinsic::type_test)); Function *ICallBranchFunnelFunc = M.getFunction(Intrinsic::getName(Intrinsic::icall_branch_funnel)); if ((!TypeTestFunc || TypeTestFunc->use_empty()) && @@ -2196,7 +2230,9 @@ bool LowerTypeTestsModule::lower() { PreservedAnalyses LowerTypeTestsPass::run(Module &M, ModuleAnalysisManager &AM) { - bool Changed = LowerTypeTestsModule(M, ExportSummary, ImportSummary).lower(); + bool Changed = + LowerTypeTestsModule(M, ExportSummary, ImportSummary, DropTypeTests) + .lower(); if (!Changed) return PreservedAnalyses::all(); return PreservedAnalyses::none(); diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp index 5ccfb29..9e8eaf3 100644 --- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -134,6 +134,22 @@ static cl::opt<bool> cl::init(false), cl::ZeroOrMore, cl::desc("Print index-based devirtualization messages")); +/// Provide a way to force enable whole program visibility in tests. +/// This is needed to support legacy tests that don't contain +/// !vcall_visibility metadata (the mere presense of type tests +/// previously implied hidden visibility). +cl::opt<bool> + WholeProgramVisibility("whole-program-visibility", cl::init(false), + cl::Hidden, cl::ZeroOrMore, + cl::desc("Enable whole program visibility")); + +/// Provide a way to force disable whole program for debugging or workarounds, +/// when enabled via the linker. +cl::opt<bool> DisableWholeProgramVisibility( + "disable-whole-program-visibility", cl::init(false), cl::Hidden, + cl::ZeroOrMore, + cl::desc("Disable whole program visibility (overrides enabling options)")); + // Find the minimum offset that we may store a value of size Size bits at. If // IsAfter is set, look for an offset before the object, otherwise look for an // offset after the object. @@ -702,7 +718,49 @@ PreservedAnalyses WholeProgramDevirtPass::run(Module &M, return PreservedAnalyses::none(); } +// Enable whole program visibility if enabled by client (e.g. linker) or +// internal option, and not force disabled. +static bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO) { + return (WholeProgramVisibilityEnabledInLTO || WholeProgramVisibility) && + !DisableWholeProgramVisibility; +} + namespace llvm { + +/// If whole program visibility asserted, then upgrade all public vcall +/// visibility metadata on vtable definitions to linkage unit visibility in +/// Module IR (for regular or hybrid LTO). +void updateVCallVisibilityInModule(Module &M, + bool WholeProgramVisibilityEnabledInLTO) { + if (!hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO)) + return; + for (GlobalVariable &GV : M.globals()) + // Add linkage unit visibility to any variable with type metadata, which are + // the vtable definitions. We won't have an existing vcall_visibility + // metadata on vtable definitions with public visibility. + if (GV.hasMetadata(LLVMContext::MD_type) && + GV.getVCallVisibility() == GlobalObject::VCallVisibilityPublic) + GV.setVCallVisibilityMetadata(GlobalObject::VCallVisibilityLinkageUnit); +} + +/// If whole program visibility asserted, then upgrade all public vcall +/// visibility metadata on vtable definition summaries to linkage unit +/// visibility in Module summary index (for ThinLTO). +void updateVCallVisibilityInIndex(ModuleSummaryIndex &Index, + bool WholeProgramVisibilityEnabledInLTO) { + if (!hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO)) + return; + for (auto &P : Index) { + for (auto &S : P.second.SummaryList) { + auto *GVar = dyn_cast<GlobalVarSummary>(S.get()); + if (!GVar || GVar->vTableFuncs().empty() || + GVar->getVCallVisibility() != GlobalObject::VCallVisibilityPublic) + continue; + GVar->setVCallVisibility(GlobalObject::VCallVisibilityLinkageUnit); + } + } +} + void runWholeProgramDevirtOnIndex( ModuleSummaryIndex &Summary, std::set<GlobalValue::GUID> &ExportedGUIDs, std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap) { @@ -818,6 +876,12 @@ bool DevirtModule::tryFindVirtualCallTargets( if (!TM.Bits->GV->isConstant()) return false; + // We cannot perform whole program devirtualization analysis on a vtable + // with public LTO visibility. + if (TM.Bits->GV->getVCallVisibility() == + GlobalObject::VCallVisibilityPublic) + return false; + Constant *Ptr = getPointerAtOffset(TM.Bits->GV->getInitializer(), TM.Offset + ByteOffset, M); if (!Ptr) @@ -863,8 +927,13 @@ bool DevirtIndex::tryFindVirtualCallTargets( return false; LocalFound = true; } - if (!GlobalValue::isAvailableExternallyLinkage(S->linkage())) + if (!GlobalValue::isAvailableExternallyLinkage(S->linkage())) { VS = cast<GlobalVarSummary>(S->getBaseObject()); + // We cannot perform whole program devirtualization analysis on a vtable + // with public LTO visibility. + if (VS->getVCallVisibility() == GlobalObject::VCallVisibilityPublic) + return false; + } } if (!VS->isLive()) continue; @@ -1808,6 +1877,12 @@ bool DevirtModule::run() { removeRedundantTypeTests(); + // We have lowered or deleted the type instrinsics, so we will no + // longer have enough information to reason about the liveness of virtual + // function pointers in GlobalDCE. + for (GlobalVariable &GV : M.globals()) + GV.eraseMetadata(LLVMContext::MD_vcall_visibility); + // The rest of the code is only necessary when exporting or during regular // LTO, so we are done. return true; @@ -1931,7 +2006,7 @@ bool DevirtModule::run() { for (VTableBits &B : Bits) rebuildGlobal(B); - // We have lowered or deleted the type checked load intrinsics, so we no + // We have lowered or deleted the type instrinsics, so we will no // longer have enough information to reason about the liveness of virtual // function pointers in GlobalDCE. for (GlobalVariable &GV : M.globals()) |