aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/LTO/LTO.cpp10
-rw-r--r--llvm/lib/LTO/LTOCodeGenerator.cpp8
-rw-r--r--llvm/lib/LTO/ThinLTOCodeGenerator.cpp6
-rw-r--r--llvm/lib/Transforms/IPO/LowerTypeTests.cpp60
-rw-r--r--llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp79
5 files changed, 149 insertions, 14 deletions
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 297b11d..e8f0fd6 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -982,6 +982,11 @@ Error LTO::runRegularLTO(AddStreamFn AddStream) {
}
}
+ // If allowed, upgrade public vcall visibility metadata to linkage unit
+ // visibility before whole program devirtualization in the optimizer.
+ updateVCallVisibilityInModule(*RegularLTO.CombinedModule,
+ Conf.HasWholeProgramVisibility);
+
if (Conf.PreOptModuleHook &&
!Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
return Error::success();
@@ -1299,6 +1304,11 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
std::set<GlobalValue::GUID> ExportedGUIDs;
+ // If allowed, upgrade public vcall visibility to linkage unit visibility in
+ // the summaries before whole program devirtualization below.
+ updateVCallVisibilityInIndex(ThinLTO.CombinedIndex,
+ Conf.HasWholeProgramVisibility);
+
// Perform index-based WPD. This will return immediately if there are
// no index entries in the typeIdMetadata map (e.g. if we are instead
// performing IR-based WPD in hybrid regular/thin LTO mode).
diff --git a/llvm/lib/LTO/LTOCodeGenerator.cpp b/llvm/lib/LTO/LTOCodeGenerator.cpp
index 5fef142..b3bc727 100644
--- a/llvm/lib/LTO/LTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/LTOCodeGenerator.cpp
@@ -57,6 +57,7 @@
#include "llvm/Transforms/IPO.h"
#include "llvm/Transforms/IPO/Internalize.h"
#include "llvm/Transforms/IPO/PassManagerBuilder.h"
+#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
#include "llvm/Transforms/ObjCARC.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <system_error>
@@ -542,6 +543,13 @@ bool LTOCodeGenerator::optimize(bool DisableVerify, bool DisableInline,
}
StatsFile = std::move(StatsFileOrErr.get());
+ // Currently there is no support for enabling whole program visibility via a
+ // linker option in the old LTO API, but this call allows it to be specified
+ // via the internal option. Must be done before WPD invoked via the optimizer
+ // pipeline run below.
+ updateVCallVisibilityInModule(*MergedModule,
+ /* WholeProgramVisibilityEnabledInLTO */ false);
+
// We always run the verifier once on the merged module, the `DisableVerify`
// parameter only applies to subsequent verify.
verifyMergedModuleOnce();
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 0bb518b..f4099e6 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -969,6 +969,12 @@ void ThinLTOCodeGenerator::run() {
// Synthesize entry counts for functions in the combined index.
computeSyntheticCounts(*Index);
+ // Currently there is no support for enabling whole program visibility via a
+ // linker option in the old LTO API, but this call allows it to be specified
+ // via the internal option. Must be done before WPD below.
+ updateVCallVisibilityInIndex(*Index,
+ /* WholeProgramVisibilityEnabledInLTO */ false);
+
// Perform index-based WPD. This will return immediately if there are
// no index entries in the typeIdMetadata map (e.g. if we are instead
// performing IR-based WPD in hybrid regular/thin LTO mode).
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index e6747a6..6eba35a 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -382,6 +382,9 @@ class LowerTypeTestsModule {
ModuleSummaryIndex *ExportSummary;
const ModuleSummaryIndex *ImportSummary;
+ // Set when the client has invoked this to simply drop all type test assume
+ // sequences.
+ bool DropTypeTests;
Triple::ArchType Arch;
Triple::OSType OS;
@@ -500,7 +503,8 @@ class LowerTypeTestsModule {
public:
LowerTypeTestsModule(Module &M, ModuleSummaryIndex *ExportSummary,
- const ModuleSummaryIndex *ImportSummary);
+ const ModuleSummaryIndex *ImportSummary,
+ bool DropTypeTests);
bool lower();
@@ -516,22 +520,24 @@ struct LowerTypeTests : public ModulePass {
ModuleSummaryIndex *ExportSummary;
const ModuleSummaryIndex *ImportSummary;
+ bool DropTypeTests;
LowerTypeTests() : ModulePass(ID), UseCommandLine(true) {
initializeLowerTypeTestsPass(*PassRegistry::getPassRegistry());
}
LowerTypeTests(ModuleSummaryIndex *ExportSummary,
- const ModuleSummaryIndex *ImportSummary)
+ const ModuleSummaryIndex *ImportSummary, bool DropTypeTests)
: ModulePass(ID), ExportSummary(ExportSummary),
- ImportSummary(ImportSummary) {
+ ImportSummary(ImportSummary), DropTypeTests(DropTypeTests) {
initializeLowerTypeTestsPass(*PassRegistry::getPassRegistry());
}
bool runOnModule(Module &M) override {
if (UseCommandLine)
return LowerTypeTestsModule::runForTesting(M);
- return LowerTypeTestsModule(M, ExportSummary, ImportSummary).lower();
+ return LowerTypeTestsModule(M, ExportSummary, ImportSummary, DropTypeTests)
+ .lower();
}
};
@@ -544,8 +550,9 @@ INITIALIZE_PASS(LowerTypeTests, "lowertypetests", "Lower type metadata", false,
ModulePass *
llvm::createLowerTypeTestsPass(ModuleSummaryIndex *ExportSummary,
- const ModuleSummaryIndex *ImportSummary) {
- return new LowerTypeTests(ExportSummary, ImportSummary);
+ const ModuleSummaryIndex *ImportSummary,
+ bool DropTypeTests) {
+ return new LowerTypeTests(ExportSummary, ImportSummary, DropTypeTests);
}
/// Build a bit set for TypeId using the object layouts in
@@ -1655,8 +1662,9 @@ void LowerTypeTestsModule::buildBitSetsFromDisjointSet(
/// Lower all type tests in this module.
LowerTypeTestsModule::LowerTypeTestsModule(
Module &M, ModuleSummaryIndex *ExportSummary,
- const ModuleSummaryIndex *ImportSummary)
- : M(M), ExportSummary(ExportSummary), ImportSummary(ImportSummary) {
+ const ModuleSummaryIndex *ImportSummary, bool DropTypeTests)
+ : M(M), ExportSummary(ExportSummary), ImportSummary(ImportSummary),
+ DropTypeTests(DropTypeTests) {
assert(!(ExportSummary && ImportSummary));
Triple TargetTriple(M.getTargetTriple());
Arch = TargetTriple.getArch();
@@ -1683,7 +1691,8 @@ bool LowerTypeTestsModule::runForTesting(Module &M) {
bool Changed =
LowerTypeTestsModule(
M, ClSummaryAction == PassSummaryAction::Export ? &Summary : nullptr,
- ClSummaryAction == PassSummaryAction::Import ? &Summary : nullptr)
+ ClSummaryAction == PassSummaryAction::Import ? &Summary : nullptr,
+ /*DropTypeTests*/ false)
.lower();
if (!ClWriteSummary.empty()) {
@@ -1750,6 +1759,33 @@ void LowerTypeTestsModule::replaceDirectCalls(Value *Old, Value *New) {
}
bool LowerTypeTestsModule::lower() {
+ Function *TypeTestFunc =
+ M.getFunction(Intrinsic::getName(Intrinsic::type_test));
+
+ if (DropTypeTests && TypeTestFunc) {
+ for (auto UI = TypeTestFunc->use_begin(), UE = TypeTestFunc->use_end();
+ UI != UE;) {
+ auto *CI = cast<CallInst>((*UI++).getUser());
+ // Find and erase llvm.assume intrinsics for this llvm.type.test call.
+ for (auto CIU = CI->use_begin(), CIUE = CI->use_end(); CIU != CIUE;) {
+ if (auto *AssumeCI = dyn_cast<CallInst>((*CIU++).getUser())) {
+ Function *F = AssumeCI->getCalledFunction();
+ if (F && F->getIntrinsicID() == Intrinsic::assume)
+ AssumeCI->eraseFromParent();
+ }
+ }
+ CI->eraseFromParent();
+ }
+
+ // We have deleted the type intrinsics, so we no longer have enough
+ // information to reason about the liveness of virtual function pointers
+ // in GlobalDCE.
+ for (GlobalVariable &GV : M.globals())
+ GV.eraseMetadata(LLVMContext::MD_vcall_visibility);
+
+ return true;
+ }
+
// If only some of the modules were split, we cannot correctly perform
// this transformation. We already checked for the presense of type tests
// with partially split modules during the thin link, and would have emitted
@@ -1758,8 +1794,6 @@ bool LowerTypeTestsModule::lower() {
(ImportSummary && ImportSummary->partiallySplitLTOUnits()))
return false;
- Function *TypeTestFunc =
- M.getFunction(Intrinsic::getName(Intrinsic::type_test));
Function *ICallBranchFunnelFunc =
M.getFunction(Intrinsic::getName(Intrinsic::icall_branch_funnel));
if ((!TypeTestFunc || TypeTestFunc->use_empty()) &&
@@ -2196,7 +2230,9 @@ bool LowerTypeTestsModule::lower() {
PreservedAnalyses LowerTypeTestsPass::run(Module &M,
ModuleAnalysisManager &AM) {
- bool Changed = LowerTypeTestsModule(M, ExportSummary, ImportSummary).lower();
+ bool Changed =
+ LowerTypeTestsModule(M, ExportSummary, ImportSummary, DropTypeTests)
+ .lower();
if (!Changed)
return PreservedAnalyses::all();
return PreservedAnalyses::none();
diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 5ccfb29..9e8eaf3 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -134,6 +134,22 @@ static cl::opt<bool>
cl::init(false), cl::ZeroOrMore,
cl::desc("Print index-based devirtualization messages"));
+/// Provide a way to force enable whole program visibility in tests.
+/// This is needed to support legacy tests that don't contain
+/// !vcall_visibility metadata (the mere presense of type tests
+/// previously implied hidden visibility).
+cl::opt<bool>
+ WholeProgramVisibility("whole-program-visibility", cl::init(false),
+ cl::Hidden, cl::ZeroOrMore,
+ cl::desc("Enable whole program visibility"));
+
+/// Provide a way to force disable whole program for debugging or workarounds,
+/// when enabled via the linker.
+cl::opt<bool> DisableWholeProgramVisibility(
+ "disable-whole-program-visibility", cl::init(false), cl::Hidden,
+ cl::ZeroOrMore,
+ cl::desc("Disable whole program visibility (overrides enabling options)"));
+
// Find the minimum offset that we may store a value of size Size bits at. If
// IsAfter is set, look for an offset before the object, otherwise look for an
// offset after the object.
@@ -702,7 +718,49 @@ PreservedAnalyses WholeProgramDevirtPass::run(Module &M,
return PreservedAnalyses::none();
}
+// Enable whole program visibility if enabled by client (e.g. linker) or
+// internal option, and not force disabled.
+static bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO) {
+ return (WholeProgramVisibilityEnabledInLTO || WholeProgramVisibility) &&
+ !DisableWholeProgramVisibility;
+}
+
namespace llvm {
+
+/// If whole program visibility asserted, then upgrade all public vcall
+/// visibility metadata on vtable definitions to linkage unit visibility in
+/// Module IR (for regular or hybrid LTO).
+void updateVCallVisibilityInModule(Module &M,
+ bool WholeProgramVisibilityEnabledInLTO) {
+ if (!hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO))
+ return;
+ for (GlobalVariable &GV : M.globals())
+ // Add linkage unit visibility to any variable with type metadata, which are
+ // the vtable definitions. We won't have an existing vcall_visibility
+ // metadata on vtable definitions with public visibility.
+ if (GV.hasMetadata(LLVMContext::MD_type) &&
+ GV.getVCallVisibility() == GlobalObject::VCallVisibilityPublic)
+ GV.setVCallVisibilityMetadata(GlobalObject::VCallVisibilityLinkageUnit);
+}
+
+/// If whole program visibility asserted, then upgrade all public vcall
+/// visibility metadata on vtable definition summaries to linkage unit
+/// visibility in Module summary index (for ThinLTO).
+void updateVCallVisibilityInIndex(ModuleSummaryIndex &Index,
+ bool WholeProgramVisibilityEnabledInLTO) {
+ if (!hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO))
+ return;
+ for (auto &P : Index) {
+ for (auto &S : P.second.SummaryList) {
+ auto *GVar = dyn_cast<GlobalVarSummary>(S.get());
+ if (!GVar || GVar->vTableFuncs().empty() ||
+ GVar->getVCallVisibility() != GlobalObject::VCallVisibilityPublic)
+ continue;
+ GVar->setVCallVisibility(GlobalObject::VCallVisibilityLinkageUnit);
+ }
+ }
+}
+
void runWholeProgramDevirtOnIndex(
ModuleSummaryIndex &Summary, std::set<GlobalValue::GUID> &ExportedGUIDs,
std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap) {
@@ -818,6 +876,12 @@ bool DevirtModule::tryFindVirtualCallTargets(
if (!TM.Bits->GV->isConstant())
return false;
+ // We cannot perform whole program devirtualization analysis on a vtable
+ // with public LTO visibility.
+ if (TM.Bits->GV->getVCallVisibility() ==
+ GlobalObject::VCallVisibilityPublic)
+ return false;
+
Constant *Ptr = getPointerAtOffset(TM.Bits->GV->getInitializer(),
TM.Offset + ByteOffset, M);
if (!Ptr)
@@ -863,8 +927,13 @@ bool DevirtIndex::tryFindVirtualCallTargets(
return false;
LocalFound = true;
}
- if (!GlobalValue::isAvailableExternallyLinkage(S->linkage()))
+ if (!GlobalValue::isAvailableExternallyLinkage(S->linkage())) {
VS = cast<GlobalVarSummary>(S->getBaseObject());
+ // We cannot perform whole program devirtualization analysis on a vtable
+ // with public LTO visibility.
+ if (VS->getVCallVisibility() == GlobalObject::VCallVisibilityPublic)
+ return false;
+ }
}
if (!VS->isLive())
continue;
@@ -1808,6 +1877,12 @@ bool DevirtModule::run() {
removeRedundantTypeTests();
+ // We have lowered or deleted the type instrinsics, so we will no
+ // longer have enough information to reason about the liveness of virtual
+ // function pointers in GlobalDCE.
+ for (GlobalVariable &GV : M.globals())
+ GV.eraseMetadata(LLVMContext::MD_vcall_visibility);
+
// The rest of the code is only necessary when exporting or during regular
// LTO, so we are done.
return true;
@@ -1931,7 +2006,7 @@ bool DevirtModule::run() {
for (VTableBits &B : Bits)
rebuildGlobal(B);
- // We have lowered or deleted the type checked load intrinsics, so we no
+ // We have lowered or deleted the type instrinsics, so we will no
// longer have enough information to reason about the liveness of virtual
// function pointers in GlobalDCE.
for (GlobalVariable &GV : M.globals())