aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Passes
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Passes')
-rw-r--r--llvm/lib/Passes/PassBuilder.cpp14
-rw-r--r--llvm/lib/Passes/PassBuilderPipelines.cpp78
-rw-r--r--llvm/lib/Passes/PassRegistry.def12
-rw-r--r--llvm/lib/Passes/StandardInstrumentations.cpp4
4 files changed, 97 insertions, 11 deletions
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 3c9a27a..f5281ea 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -45,7 +45,6 @@
#include "llvm/Analysis/IR2Vec.h"
#include "llvm/Analysis/IVUsers.h"
#include "llvm/Analysis/InlineAdvisor.h"
-#include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
#include "llvm/Analysis/InstCount.h"
#include "llvm/Analysis/KernelInfo.h"
#include "llvm/Analysis/LastRunTrackingAnalysis.h"
@@ -67,6 +66,7 @@
#include "llvm/Analysis/PostDominators.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/RegionInfo.h"
+#include "llvm/Analysis/RuntimeLibcallInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
#include "llvm/Analysis/ScalarEvolutionDivision.h"
@@ -899,6 +899,11 @@ Expected<bool> parseEntryExitInstrumenterPassOptions(StringRef Params) {
"EntryExitInstrumenter");
}
+Expected<bool> parseDropUnnecessaryAssumesPassOptions(StringRef Params) {
+ return PassBuilder::parseSinglePassOption(Params, "drop-deref",
+ "DropUnnecessaryAssumes");
+}
+
Expected<bool> parseLoopExtractorPassOptions(StringRef Params) {
return PassBuilder::parseSinglePassOption(Params, "single", "LoopExtractor");
}
@@ -1585,24 +1590,31 @@ parseBoundsCheckingOptions(StringRef Params) {
Options.Rt = {
/*MinRuntime=*/false,
/*MayReturn=*/true,
+ /*HandlerPreserveAllRegs=*/false,
};
} else if (ParamName == "rt-abort") {
Options.Rt = {
/*MinRuntime=*/false,
/*MayReturn=*/false,
+ /*HandlerPreserveAllRegs=*/false,
};
} else if (ParamName == "min-rt") {
Options.Rt = {
/*MinRuntime=*/true,
/*MayReturn=*/true,
+ /*HandlerPreserveAllRegs=*/false,
};
} else if (ParamName == "min-rt-abort") {
Options.Rt = {
/*MinRuntime=*/true,
/*MayReturn=*/false,
+ /*HandlerPreserveAllRegs=*/false,
};
} else if (ParamName == "merge") {
Options.Merge = true;
+ } else if (ParamName == "handler-preserve-all-regs") {
+ if (Options.Rt)
+ Options.Rt->HandlerPreserveAllRegs = true;
} else {
StringRef ParamEQ;
StringRef Val;
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index bd03ac0..4de527d 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -73,6 +73,7 @@
#include "llvm/Transforms/IPO/SampleProfileProbe.h"
#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
#include "llvm/Transforms/InstCombine/InstCombine.h"
+#include "llvm/Transforms/Instrumentation/AllocToken.h"
#include "llvm/Transforms/Instrumentation/CGProfile.h"
#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
@@ -304,6 +305,13 @@ static cl::opt<std::string> InstrumentColdFuncOnlyPath(
"with --pgo-instrument-cold-function-only)"),
cl::Hidden);
+// TODO: There is a similar flag in WPD pass, we should consolidate them by
+// parsing the option only once in PassBuilder and share it across both places.
+static cl::opt<bool> EnableDevirtualizeSpeculatively(
+ "enable-devirtualize-speculatively",
+ cl::desc("Enable speculative devirtualization optimization"),
+ cl::init(false));
+
extern cl::opt<std::string> UseCtxProfile;
extern cl::opt<bool> PGOInstrumentColdFunctionOnly;
@@ -325,6 +333,7 @@ PipelineTuningOptions::PipelineTuningOptions() {
MergeFunctions = EnableMergeFunctions;
InlinerThreshold = -1;
EagerlyInvalidateAnalyses = EnableEagerlyInvalidateAnalyses;
+ DevirtualizeSpeculatively = EnableDevirtualizeSpeculatively;
}
namespace llvm {
@@ -1298,10 +1307,18 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
/// TODO: Should LTO cause any differences to this set of passes?
void PassBuilder::addVectorPasses(OptimizationLevel Level,
- FunctionPassManager &FPM, bool IsFullLTO) {
+ FunctionPassManager &FPM,
+ ThinOrFullLTOPhase LTOPhase) {
+ const bool IsFullLTO = LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink;
+
FPM.addPass(LoopVectorizePass(
LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
+ // Drop dereferenceable assumes after vectorization, as they are no longer
+ // needed and can inhibit further optimization.
+ if (!isLTOPreLink(LTOPhase))
+ FPM.addPass(DropUnnecessaryAssumesPass(/*DropDereferenceable=*/true));
+
FPM.addPass(InferAlignmentPass());
if (IsFullLTO) {
// The vectorizer may have significantly shortened a loop body; unroll
@@ -1572,7 +1589,7 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
// from the TargetLibraryInfo.
OptimizePM.addPass(InjectTLIMappings());
- addVectorPasses(Level, OptimizePM, /* IsFullLTO */ false);
+ addVectorPasses(Level, OptimizePM, LTOPhase);
invokeVectorizerEndEPCallbacks(OptimizePM, Level);
@@ -1607,6 +1624,11 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM),
PTO.EagerlyInvalidateAnalyses));
+ // AllocToken transforms heap allocation calls; this needs to run late after
+ // other allocation call transformations (such as those in InstCombine).
+ if (!LTOPreLink)
+ MPM.addPass(AllocTokenPass());
+
invokeOptimizerLastEPCallbacks(MPM, Level, LTOPhase);
// Split out cold code. Splitting is done late to avoid hiding context from
@@ -1641,6 +1663,34 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
if (!LTOPreLink)
MPM.addPass(RelLookupTableConverterPass());
+ // Add devirtualization pass only when LTO is not enabled, as otherwise
+ // the pass is already enabled in the LTO pipeline.
+ if (PTO.DevirtualizeSpeculatively && LTOPhase == ThinOrFullLTOPhase::None) {
+ // TODO: explore a better pipeline configuration that can improve
+ // compilation time overhead.
+ MPM.addPass(WholeProgramDevirtPass(
+ /*ExportSummary*/ nullptr,
+ /*ImportSummary*/ nullptr,
+ /*DevirtSpeculatively*/ PTO.DevirtualizeSpeculatively));
+ MPM.addPass(LowerTypeTestsPass(nullptr, nullptr,
+ lowertypetests::DropTestKind::Assume));
+ // Given that the devirtualization creates more opportunities for inlining,
+ // we run the Inliner again here to maximize the optimization gain we
+ // get from devirtualization.
+ // Also, we can't run devirtualization before inlining because the
+ // devirtualization depends on the passes optimizing/eliminating vtable GVs
+ // and those passes are only effective after inlining.
+ if (EnableModuleInliner) {
+ MPM.addPass(ModuleInlinerPass(getInlineParamsFromOptLevel(Level),
+ UseInlineAdvisor,
+ ThinOrFullLTOPhase::None));
+ } else {
+ MPM.addPass(ModuleInlinerWrapperPass(
+ getInlineParamsFromOptLevel(Level),
+ /* MandatoryFirst */ true,
+ InlineContext{ThinOrFullLTOPhase::None, InlinePass::CGSCCInliner}));
+ }
+ }
return MPM;
}
@@ -1845,6 +1895,11 @@ ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
MPM.addPass(LowerTypeTestsPass(nullptr, nullptr,
lowertypetests::DropTestKind::Assume));
MPM.addPass(buildCoroWrapper(ThinOrFullLTOPhase::ThinLTOPostLink));
+
+ // AllocToken transforms heap allocation calls; this needs to run late after
+ // other allocation call transformations (such as those in InstCombine).
+ MPM.addPass(AllocTokenPass());
+
// Drop available_externally and unreferenced globals. This is necessary
// with ThinLTO in order to avoid leaving undefined references to dead
// globals in the object file.
@@ -1906,6 +1961,10 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
MPM.addPass(buildCoroWrapper(ThinOrFullLTOPhase::FullLTOPostLink));
+ // AllocToken transforms heap allocation calls; this needs to run late after
+ // other allocation call transformations (such as those in InstCombine).
+ MPM.addPass(AllocTokenPass());
+
invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
// Emit annotation remarks.
@@ -1993,6 +2052,10 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
MPM.addPass(buildCoroWrapper(ThinOrFullLTOPhase::FullLTOPostLink));
+ // AllocToken transforms heap allocation calls; this needs to run late after
+ // other allocation call transformations (such as those in InstCombine).
+ MPM.addPass(AllocTokenPass());
+
invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
// Emit annotation remarks.
@@ -2162,7 +2225,7 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
MainFPM.addPass(LoopDistributePass());
- addVectorPasses(Level, MainFPM, /* IsFullLTO */ true);
+ addVectorPasses(Level, MainFPM, ThinOrFullLTOPhase::FullLTOPostLink);
invokeVectorizerEndEPCallbacks(MainFPM, Level);
@@ -2227,6 +2290,10 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
MPM.addPass(CoroCleanupPass());
+ // AllocToken transforms heap allocation calls; this needs to run late after
+ // other allocation call transformations (such as those in InstCombine).
+ MPM.addPass(AllocTokenPass());
+
invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
// Emit annotation remarks.
@@ -2343,6 +2410,11 @@ PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
MPM.addPass(buildCoroWrapper(Phase));
+ // AllocToken transforms heap allocation calls; this needs to run late after
+ // other allocation call transformations (such as those in InstCombine).
+ if (!isLTOPreLink(Phase))
+ MPM.addPass(AllocTokenPass());
+
invokeOptimizerLastEPCallbacks(MPM, Level, Phase);
if (isLTOPreLink(Phase))
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 1853cdd..cf998f2 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -30,11 +30,13 @@ MODULE_ANALYSIS("ir2vec-vocab", IR2VecVocabAnalysis())
MODULE_ANALYSIS("ir-similarity", IRSimilarityAnalysis())
MODULE_ANALYSIS("last-run-tracking", LastRunTrackingAnalysis())
MODULE_ANALYSIS("lcg", LazyCallGraphAnalysis())
+MODULE_ANALYSIS("libcall-lowering-info", LibcallLoweringModuleAnalysis())
MODULE_ANALYSIS("module-summary", ModuleSummaryIndexAnalysis())
MODULE_ANALYSIS("no-op-module", NoOpModuleAnalysis())
MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
MODULE_ANALYSIS("profile-summary", ProfileSummaryAnalysis())
MODULE_ANALYSIS("reg-usage", PhysicalRegisterUsageAnalysis())
+MODULE_ANALYSIS("runtime-libcall-info", RuntimeLibraryAnalysis())
MODULE_ANALYSIS("stack-safety", StackSafetyGlobalAnalysis())
MODULE_ANALYSIS("verify", VerifierAnalysis())
@@ -147,6 +149,7 @@ MODULE_PASS("print<ir2vec>", IR2VecPrinterPass(errs()))
MODULE_PASS("print<ir2vec-vocab>", IR2VecVocabPrinterPass(errs()))
MODULE_PASS("print<module-debuginfo>", ModuleDebugInfoPrinterPass(errs()))
MODULE_PASS("print<reg-usage>", PhysicalRegisterUsageInfoPrinterPass(errs()))
+MODULE_PASS("prof-verify", ProfileVerifierPass())
MODULE_PASS("pseudo-probe", SampleProfileProbePass(TM))
MODULE_PASS("pseudo-probe-update", PseudoProbeUpdatePass())
MODULE_PASS("recompute-globalsaa", RecomputeGlobalsAAPass())
@@ -358,7 +361,6 @@ FUNCTION_ANALYSIS("ephemerals", EphemeralValuesAnalysis())
FUNCTION_ANALYSIS("func-properties", FunctionPropertiesAnalysis())
FUNCTION_ANALYSIS("machine-function-info", MachineFunctionAnalysis(*TM))
FUNCTION_ANALYSIS("gc-function", GCFunctionAnalysis())
-FUNCTION_ANALYSIS("inliner-size-estimator", InlineSizeEstimatorAnalysis())
FUNCTION_ANALYSIS("last-run-tracking", LastRunTrackingAnalysis())
FUNCTION_ANALYSIS("lazy-value-info", LazyValueAnalysis())
FUNCTION_ANALYSIS("loops", LoopAnalysis())
@@ -431,7 +433,6 @@ FUNCTION_PASS("dot-post-dom", PostDomPrinter())
FUNCTION_PASS("dot-post-dom-only", PostDomOnlyPrinter())
FUNCTION_PASS("dse", DSEPass())
FUNCTION_PASS("dwarf-eh-prepare", DwarfEHPreparePass(*TM))
-FUNCTION_PASS("drop-unnecessary-assumes", DropUnnecessaryAssumesPass())
FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass(*TM))
FUNCTION_PASS("expand-memcmp", ExpandMemCmpPass(*TM))
FUNCTION_PASS("expand-reductions", ExpandReductionsPass())
@@ -515,8 +516,6 @@ FUNCTION_PASS("print<domfrontier>", DominanceFrontierPrinterPass(errs()))
FUNCTION_PASS("print<domtree>", DominatorTreePrinterPass(errs()))
FUNCTION_PASS("print<func-properties>", FunctionPropertiesPrinterPass(errs()))
FUNCTION_PASS("print<inline-cost>", InlineCostAnnotationPrinterPass(errs()))
-FUNCTION_PASS("print<inliner-size-estimator>",
- InlineSizeEstimatorAnalysisPrinterPass(errs()))
FUNCTION_PASS("print<lazy-value-info>", LazyValueInfoPrinterPass(errs()))
FUNCTION_PASS("print<loops>", LoopPrinterPass(errs()))
FUNCTION_PASS("print<memoryssa-walker>", MemorySSAWalkerPrinterPass(errs()))
@@ -528,7 +527,6 @@ FUNCTION_PASS("print<scev-division>", SCEVDivisionPrinterPass(errs()))
FUNCTION_PASS("print<stack-safety-local>", StackSafetyPrinterPass(errs()))
FUNCTION_PASS("print<uniformity>", UniformityInfoPrinterPass(errs()))
FUNCTION_PASS("prof-inject", ProfileInjectorPass())
-FUNCTION_PASS("prof-verify", ProfileVerifierPass())
FUNCTION_PASS("reassociate", ReassociatePass())
FUNCTION_PASS("redundant-dbg-inst-elim", RedundantDbgInstEliminationPass())
FUNCTION_PASS("replace-with-veclib", ReplaceWithVeclib())
@@ -584,6 +582,10 @@ FUNCTION_PASS_WITH_PARAMS(
[](bool UseMemorySSA) { return EarlyCSEPass(UseMemorySSA); },
parseEarlyCSEPassOptions, "memssa")
FUNCTION_PASS_WITH_PARAMS(
+ "drop-unnecessary-assumes", "DropUnnecessaryAssumesPass",
+ [](bool DropDereferenceable) { return DropUnnecessaryAssumesPass(DropDereferenceable); },
+ parseDropUnnecessaryAssumesPassOptions, "drop-deref")
+FUNCTION_PASS_WITH_PARAMS(
"ee-instrument", "EntryExitInstrumenterPass",
[](bool PostInlining) { return EntryExitInstrumenterPass(PostInlining); },
parseEntryExitInstrumenterPassOptions, "post-inline")
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 7290a86..6b7e980 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -537,7 +537,7 @@ void IRChangedPrinter::handleAfter(StringRef PassID, std::string &Name,
Out << "*** IR Dump After " << PassID << " on " << Name << " ***\n" << After;
}
-IRChangedTester::~IRChangedTester() {}
+IRChangedTester::~IRChangedTester() = default;
void IRChangedTester::registerCallbacks(PassInstrumentationCallbacks &PIC) {
if (TestChanged != "")
@@ -1566,7 +1566,7 @@ void InLineChangePrinter::registerCallbacks(PassInstrumentationCallbacks &PIC) {
TextChangeReporter<IRDataT<EmptyData>>::registerRequiredCallbacks(PIC);
}
-TimeProfilingPassesHandler::TimeProfilingPassesHandler() {}
+TimeProfilingPassesHandler::TimeProfilingPassesHandler() = default;
void TimeProfilingPassesHandler::registerCallbacks(
PassInstrumentationCallbacks &PIC) {
generated by cgit v1.1 at 2026-01-06 17:42:45 +0000