aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms')
-rw-r--r--llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp16
-rw-r--r--llvm/lib/Transforms/Coroutines/CoroFrame.cpp12
-rw-r--r--llvm/lib/Transforms/Instrumentation/AllocToken.cpp82
-rw-r--r--llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp13
-rw-r--r--llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp16
-rw-r--r--llvm/lib/Transforms/Utils/Debugify.cpp5
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp2
7 files changed, 111 insertions, 35 deletions
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 805bdb4..bbbac45 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -28,8 +28,12 @@
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/ProfDataUtils.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/BuildLibCalls.h"
#include "llvm/Transforms/Utils/Local.h"
@@ -39,6 +43,10 @@ using namespace PatternMatch;
#define DEBUG_TYPE "aggressive-instcombine"
+namespace llvm {
+extern cl::opt<bool> ProfcheckDisableMetadataFixes;
+}
+
STATISTIC(NumAnyOrAllBitsSet, "Number of any/all-bits-set patterns folded");
STATISTIC(NumGuardedRotates,
"Number of guarded rotates transformed into funnel shifts");
@@ -599,6 +607,14 @@ static bool tryToRecognizeTableBasedCttz(Instruction &I, const DataLayout &DL) {
auto Cmp = B.CreateICmpEQ(X1, ConstantInt::get(XType, 0));
auto Select = B.CreateSelect(Cmp, B.CreateZExt(ZeroTableElem, XType), Cttz);
+ // The true branch of select handles the cttz(0) case, which is rare.
+ if (!ProfcheckDisableMetadataFixes) {
+ if (Instruction *SelectI = dyn_cast<Instruction>(Select))
+ SelectI->setMetadata(
+ LLVMContext::MD_prof,
+ MDBuilder(SelectI->getContext()).createUnlikelyBranchWeights());
+ }
+
// NOTE: If the table[0] is 0, but the cttz(0) is defined by the Target
// it should be handled as: `cttz(x) & (typeSize - 1)`.
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 0accb22..c89af68 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -689,10 +689,14 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
DISubprogram *DIS = F.getSubprogram();
// If there is no DISubprogram for F, it implies the function is compiled
// without debug info. So we also don't generate debug info for the frame.
- if (!DIS || !DIS->getUnit() ||
- !dwarf::isCPlusPlus(
- (dwarf::SourceLanguage)DIS->getUnit()->getSourceLanguage()) ||
- DIS->getUnit()->getEmissionKind() != DICompileUnit::DebugEmissionKind::FullDebug)
+
+ if (!DIS || !DIS->getUnit())
+ return;
+
+ if (!dwarf::isCPlusPlus(static_cast<llvm::dwarf::SourceLanguage>(
+ DIS->getUnit()->getSourceLanguage().getUnversionedName())) ||
+ DIS->getUnit()->getEmissionKind() !=
+ DICompileUnit::DebugEmissionKind::FullDebug)
return;
assert(Shape.ABI == coro::ABI::Switch &&
diff --git a/llvm/lib/Transforms/Instrumentation/AllocToken.cpp b/llvm/lib/Transforms/Instrumentation/AllocToken.cpp
index 782d5a1..40720ae 100644
--- a/llvm/lib/Transforms/Instrumentation/AllocToken.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AllocToken.cpp
@@ -69,19 +69,30 @@ enum class TokenMode : unsigned {
/// Token ID based on allocated type hash.
TypeHash = 2,
+
+ /// Token ID based on allocated type hash, where the top half ID-space is
+ /// reserved for types that contain pointers and the bottom half for types
+ /// that do not contain pointers.
+ TypeHashPointerSplit = 3,
};
//===--- Command-line options ---------------------------------------------===//
-cl::opt<TokenMode>
- ClMode("alloc-token-mode", cl::Hidden, cl::desc("Token assignment mode"),
- cl::init(TokenMode::TypeHash),
- cl::values(clEnumValN(TokenMode::Increment, "increment",
- "Incrementally increasing token ID"),
- clEnumValN(TokenMode::Random, "random",
- "Statically-assigned random token ID"),
- clEnumValN(TokenMode::TypeHash, "typehash",
- "Token ID based on allocated type hash")));
+cl::opt<TokenMode> ClMode(
+ "alloc-token-mode", cl::Hidden, cl::desc("Token assignment mode"),
+ cl::init(TokenMode::TypeHashPointerSplit),
+ cl::values(
+ clEnumValN(TokenMode::Increment, "increment",
+ "Incrementally increasing token ID"),
+ clEnumValN(TokenMode::Random, "random",
+ "Statically-assigned random token ID"),
+ clEnumValN(TokenMode::TypeHash, "typehash",
+ "Token ID based on allocated type hash"),
+ clEnumValN(
+ TokenMode::TypeHashPointerSplit, "typehashpointersplit",
+ "Token ID based on allocated type hash, where the top half "
+ "ID-space is reserved for types that contain pointers and the "
+ "bottom half for types that do not contain pointers. ")));
cl::opt<std::string> ClFuncPrefix("alloc-token-prefix",
cl::desc("The allocation function prefix"),
@@ -127,16 +138,23 @@ STATISTIC(NumAllocationsInstrumented, "Allocations instrumented");
/// Returns the !alloc_token metadata if available.
///
-/// Expected format is: !{<type-name>}
+/// Expected format is: !{<type-name>, <contains-pointer>}
MDNode *getAllocTokenMetadata(const CallBase &CB) {
MDNode *Ret = CB.getMetadata(LLVMContext::MD_alloc_token);
if (!Ret)
return nullptr;
- assert(Ret->getNumOperands() == 1 && "bad !alloc_token");
+ assert(Ret->getNumOperands() == 2 && "bad !alloc_token");
assert(isa<MDString>(Ret->getOperand(0)));
+ assert(isa<ConstantAsMetadata>(Ret->getOperand(1)));
return Ret;
}
+bool containsPointer(const MDNode *MD) {
+ ConstantAsMetadata *C = cast<ConstantAsMetadata>(MD->getOperand(1));
+ auto *CI = cast<ConstantInt>(C->getValue());
+ return CI->getValue().getBoolValue();
+}
+
class ModeBase {
public:
explicit ModeBase(const IntegerType &TokenTy, uint64_t MaxTokens)
@@ -188,12 +206,20 @@ public:
using ModeBase::ModeBase;
uint64_t operator()(const CallBase &CB, OptimizationRemarkEmitter &ORE) {
+ const auto [N, H] = getHash(CB, ORE);
+ return N ? boundedToken(H) : H;
+ }
+
+protected:
+ std::pair<MDNode *, uint64_t> getHash(const CallBase &CB,
+ OptimizationRemarkEmitter &ORE) {
if (MDNode *N = getAllocTokenMetadata(CB)) {
MDString *S = cast<MDString>(N->getOperand(0));
- return boundedToken(getStableSipHash(S->getString()));
+ return {N, getStableSipHash(S->getString())};
}
+ // Fallback.
remarkNoMetadata(CB, ORE);
- return ClFallbackToken;
+ return {nullptr, ClFallbackToken};
}
/// Remark that there was no precise type information.
@@ -210,6 +236,29 @@ public:
}
};
+/// Implementation for TokenMode::TypeHashPointerSplit.
+class TypeHashPointerSplitMode : public TypeHashMode {
+public:
+ using TypeHashMode::TypeHashMode;
+
+ uint64_t operator()(const CallBase &CB, OptimizationRemarkEmitter &ORE) {
+ if (MaxTokens == 1)
+ return 0;
+ const uint64_t HalfTokens = MaxTokens / 2;
+ const auto [N, H] = getHash(CB, ORE);
+ if (!N) {
+ // Pick the fallback token (ClFallbackToken), which by default is 0,
+ // meaning it'll fall into the pointer-less bucket. Override by setting
+ // -alloc-token-fallback if that is the wrong choice.
+ return H;
+ }
+ uint64_t Hash = H % HalfTokens; // base hash
+ if (containsPointer(N))
+ Hash += HalfTokens;
+ return Hash;
+ }
+};
+
// Apply opt overrides.
AllocTokenOptions transformOptionsFromCl(AllocTokenOptions Opts) {
if (!Opts.MaxTokens.has_value())
@@ -236,6 +285,9 @@ public:
case TokenMode::TypeHash:
Mode.emplace<TypeHashMode>(*IntPtrTy, *Options.MaxTokens);
break;
+ case TokenMode::TypeHashPointerSplit:
+ Mode.emplace<TypeHashPointerSplitMode>(*IntPtrTy, *Options.MaxTokens);
+ break;
}
}
@@ -275,7 +327,9 @@ private:
// Cache for replacement functions.
DenseMap<std::pair<LibFunc, uint64_t>, FunctionCallee> TokenAllocFunctions;
// Selected mode.
- std::variant<IncrementMode, RandomMode, TypeHashMode> Mode;
+ std::variant<IncrementMode, RandomMode, TypeHashMode,
+ TypeHashPointerSplitMode>
+ Mode;
};
bool AllocToken::instrumentFunction(Function &F) {
diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index 5b8ea15..b74a070 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -1084,8 +1084,10 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
auto ThenTerm = SplitBlockAndInsertIfThen(
IRB.CreateIsNull(Load), &*IP, false,
MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
- IRBuilder<> ThenIRB(ThenTerm);
+ InstrumentationIRBuilder ThenIRB(ThenTerm);
auto Store = ThenIRB.CreateStore(ConstantInt::getTrue(Int1Ty), FlagPtr);
+ if (EntryLoc)
+ Store->setDebugLoc(EntryLoc);
Load->setNoSanitizeMetadata();
Store->setNoSanitizeMetadata();
}
@@ -1131,7 +1133,10 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
EstimatedStackSize >= Options.StackDepthCallbackMin) {
if (InsertBefore)
IRB.SetInsertPoint(InsertBefore);
- IRB.CreateCall(SanCovStackDepthCallback)->setCannotMerge();
+ auto Call = IRB.CreateCall(SanCovStackDepthCallback);
+ if (EntryLoc)
+ Call->setDebugLoc(EntryLoc);
+ Call->setCannotMerge();
}
} else {
// Check stack depth. If it's the deepest so far, record it.
@@ -1144,8 +1149,10 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
auto ThenTerm = SplitBlockAndInsertIfThen(
IsStackLower, &*IP, false,
MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
- IRBuilder<> ThenIRB(ThenTerm);
+ InstrumentationIRBuilder ThenIRB(ThenTerm);
auto Store = ThenIRB.CreateStore(FrameAddrInt, SanCovLowestStack);
+ if (EntryLoc)
+ Store->setDebugLoc(EntryLoc);
LowestStack->setNoSanitizeMetadata();
Store->setNoSanitizeMetadata();
}
diff --git a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
index e448230..3f7003d 100644
--- a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
@@ -61,6 +61,7 @@
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringExtras.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/DomTreeUpdater.h"
@@ -382,16 +383,9 @@ typedef DenseMap<BasicBlock *, CloneList> DuplicateBlockMap;
typedef MapVector<Instruction *, std::vector<Instruction *>> DefMap;
inline raw_ostream &operator<<(raw_ostream &OS, const PathType &Path) {
- OS << "< ";
- for (const BasicBlock *BB : Path) {
- std::string BBName;
- if (BB->hasName())
- raw_string_ostream(BBName) << BB->getName();
- else
- raw_string_ostream(BBName) << BB;
- OS << BBName << " ";
- }
- OS << ">";
+ auto BBNames = llvm::map_range(
+ Path, [](const BasicBlock *BB) { return BB->getNameOrAsOperand(); });
+ OS << "< " << llvm::join(BBNames, ", ") << " >";
return OS;
}
@@ -423,7 +417,7 @@ struct ThreadingPath {
}
void print(raw_ostream &OS) const {
- OS << Path << " [ " << ExitVal << ", " << DBB->getName() << " ]";
+ OS << Path << " [ " << ExitVal << ", " << DBB->getNameOrAsOperand() << " ]";
}
private:
diff --git a/llvm/lib/Transforms/Utils/Debugify.cpp b/llvm/lib/Transforms/Utils/Debugify.cpp
index 5a09b73..2923633 100644
--- a/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -19,6 +19,7 @@
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/DIBuilder.h"
#include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instructions.h"
@@ -162,8 +163,8 @@ bool llvm::applyDebugifyMetadata(
unsigned NextLine = 1;
unsigned NextVar = 1;
auto File = DIB.createFile(M.getName(), "/");
- auto CU = DIB.createCompileUnit(dwarf::DW_LANG_C, File, "debugify",
- /*isOptimized=*/true, "", 0);
+ auto CU = DIB.createCompileUnit(DISourceLanguageName(dwarf::DW_LANG_C), File,
+ "debugify", /*isOptimized=*/true, "", 0);
// Visit each instruction.
for (Function &F : Functions) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 3a9770c..600ff8a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3141,7 +3141,7 @@ static bool isUsedByLoadStoreAddress(const VPUser *V) {
while (!WorkList.empty()) {
auto *Cur = dyn_cast<VPSingleDefRecipe>(WorkList.pop_back_val());
- if (!Cur || !Seen.insert(Cur).second)
+ if (!Cur || !Seen.insert(Cur).second || isa<VPBlendRecipe>(Cur))
continue;
for (VPUser *U : Cur->users()) {
generated by cgit v1.1 at 2025-10-09 11:40:57 +0000