aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/IR
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/IR')
-rw-r--r--llvm/lib/IR/AsmWriter.cpp4
-rw-r--r--llvm/lib/IR/Assumptions.cpp22
-rw-r--r--llvm/lib/IR/Attributes.cpp13
-rw-r--r--llvm/lib/IR/AutoUpgrade.cpp38
-rw-r--r--llvm/lib/IR/ConstantFold.cpp1
-rw-r--r--llvm/lib/IR/ConstantRange.cpp42
-rw-r--r--llvm/lib/IR/Constants.cpp20
-rw-r--r--llvm/lib/IR/Core.cpp10
-rw-r--r--llvm/lib/IR/DataLayout.cpp52
-rw-r--r--llvm/lib/IR/DebugInfo.cpp71
-rw-r--r--llvm/lib/IR/DebugInfoMetadata.cpp10
-rw-r--r--llvm/lib/IR/Globals.cpp1
-rw-r--r--llvm/lib/IR/IRBuilder.cpp20
-rw-r--r--llvm/lib/IR/Instruction.cpp32
-rw-r--r--llvm/lib/IR/Instructions.cpp52
-rw-r--r--llvm/lib/IR/IntrinsicInst.cpp5
-rw-r--r--llvm/lib/IR/LLVMContextImpl.h27
-rw-r--r--llvm/lib/IR/Mangler.cpp3
-rw-r--r--llvm/lib/IR/ProfDataUtils.cpp12
-rw-r--r--llvm/lib/IR/RuntimeLibcalls.cpp194
-rw-r--r--llvm/lib/IR/Type.cpp11
-rw-r--r--llvm/lib/IR/Value.cpp49
-rw-r--r--llvm/lib/IR/Verifier.cpp124
23 files changed, 452 insertions, 361 deletions
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index e5a4e1e..094678f 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -88,6 +88,8 @@
using namespace llvm;
+// See https://llvm.org/docs/DebuggingLLVM.html for why these flags are useful.
+
static cl::opt<bool>
PrintInstAddrs("print-inst-addrs", cl::Hidden,
cl::desc("Print addresses of instructions when dumping"));
@@ -1163,7 +1165,7 @@ int SlotTracker::processIndex() {
std::vector<StringRef> ModulePaths;
for (auto &[ModPath, _] : TheIndex->modulePaths())
ModulePaths.push_back(ModPath);
- llvm::sort(ModulePaths.begin(), ModulePaths.end());
+ llvm::sort(ModulePaths);
for (auto &ModPath : ModulePaths)
CreateModulePathSlot(ModPath);
diff --git a/llvm/lib/IR/Assumptions.cpp b/llvm/lib/IR/Assumptions.cpp
index 6adbbc4..f8bbcb3 100644
--- a/llvm/lib/IR/Assumptions.cpp
+++ b/llvm/lib/IR/Assumptions.cpp
@@ -101,12 +101,16 @@ bool llvm::addAssumptions(CallBase &CB,
return ::addAssumptionsImpl(CB, Assumptions);
}
-StringSet<> llvm::KnownAssumptionStrings({
- "omp_no_openmp", // OpenMP 5.1
- "omp_no_openmp_routines", // OpenMP 5.1
- "omp_no_parallelism", // OpenMP 5.1
- "omp_no_openmp_constructs", // OpenMP 6.0
- "ompx_spmd_amenable", // OpenMPOpt extension
- "ompx_no_call_asm", // OpenMPOpt extension
- "ompx_aligned_barrier", // OpenMPOpt extension
-});
+StringSet<> &llvm::getKnownAssumptionStrings() {
+ static StringSet<> Object({
+ "omp_no_openmp", // OpenMP 5.1
+ "omp_no_openmp_routines", // OpenMP 5.1
+ "omp_no_parallelism", // OpenMP 5.1
+ "omp_no_openmp_constructs", // OpenMP 6.0
+ "ompx_spmd_amenable", // OpenMPOpt extension
+ "ompx_no_call_asm", // OpenMPOpt extension
+ "ompx_aligned_barrier", // OpenMPOpt extension
+ });
+
+ return Object;
+}
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index d1fbcb9..4ac2ebd 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -954,6 +954,19 @@ AttributeSet AttributeSet::addAttributes(LLVMContext &C,
return get(C, B);
}
+AttributeSet AttributeSet::addAttributes(LLVMContext &C,
+ const AttrBuilder &B) const {
+ if (!hasAttributes())
+ return get(C, B);
+
+ if (!B.hasAttributes())
+ return *this;
+
+ AttrBuilder Merged(C, *this);
+ Merged.merge(B);
+ return get(C, Merged);
+}
+
AttributeSet AttributeSet::removeAttribute(LLVMContext &C,
Attribute::AttrKind Kind) const {
if (!hasAttribute(Kind)) return *this;
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 7159107..7ea9c6d 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1311,14 +1311,15 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
}
break;
case 'l':
- if (Name.starts_with("lifetime.start") ||
- Name.starts_with("lifetime.end")) {
- // Unless remangling is required, do not upgrade the function declaration,
- // but do upgrade the calls.
- if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F))
- NewFn = *Result;
- else
- NewFn = F;
+ if ((Name.starts_with("lifetime.start") ||
+ Name.starts_with("lifetime.end")) &&
+ F->arg_size() == 2) {
+ Intrinsic::ID IID = Name.starts_with("lifetime.start")
+ ? Intrinsic::lifetime_start
+ : Intrinsic::lifetime_end;
+ rename(F);
+ NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
+ F->getArg(0)->getType());
return true;
}
break;
@@ -5133,21 +5134,20 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end: {
- Value *Size = CI->getArgOperand(0);
- Value *Ptr = CI->getArgOperand(1);
- if (isa<AllocaInst>(Ptr)) {
+ if (CI->arg_size() != 2) {
DefaultCase();
return;
}
+ Value *Ptr = CI->getArgOperand(1);
// Try to strip pointer casts, such that the lifetime works on an alloca.
Ptr = Ptr->stripPointerCasts();
if (isa<AllocaInst>(Ptr)) {
// Don't use NewFn, as we might have looked through an addrspacecast.
if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
- NewCall = Builder.CreateLifetimeStart(Ptr, cast<ConstantInt>(Size));
+ NewCall = Builder.CreateLifetimeStart(Ptr);
else
- NewCall = Builder.CreateLifetimeEnd(Ptr, cast<ConstantInt>(Size));
+ NewCall = Builder.CreateLifetimeEnd(Ptr);
break;
}
@@ -5381,6 +5381,16 @@ bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
upgradeNVVMFnVectorAttr("nvvm.cluster_dim", K[0], GV, V);
return true;
}
+ if (K == "grid_constant") {
+ const auto Attr = Attribute::get(GV->getContext(), "nvvm.grid_constant");
+ for (const auto &Op : cast<MDNode>(V)->operands()) {
+ // For some reason, the index is 1-based in the metadata. Good thing we're
+ // able to auto-upgrade it!
+ const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
+ cast<Function>(GV)->addParamAttr(Index, Attr);
+ }
+ return true;
+ }
return false;
}
@@ -5391,7 +5401,7 @@ void llvm::UpgradeNVVMAnnotations(Module &M) {
return;
SmallVector<MDNode *, 8> NewNodes;
- SmallSet<const MDNode *, 8> SeenNodes;
+ SmallPtrSet<const MDNode *, 8> SeenNodes;
for (MDNode *MD : NamedMD->operands()) {
if (!SeenNodes.insert(MD).second)
continue;
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index d4ad21e..6b202ba 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -254,6 +254,7 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
return FoldBitCast(V, DestTy);
case Instruction::AddrSpaceCast:
case Instruction::IntToPtr:
+ case Instruction::PtrToAddr:
case Instruction::PtrToInt:
return nullptr;
}
diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp
index e09c139..b454c9a 100644
--- a/llvm/lib/IR/ConstantRange.cpp
+++ b/llvm/lib/IR/ConstantRange.cpp
@@ -829,6 +829,7 @@ ConstantRange ConstantRange::castOp(Instruction::CastOps CastOp,
case Instruction::FPTrunc:
case Instruction::FPExt:
case Instruction::IntToPtr:
+ case Instruction::PtrToAddr:
case Instruction::PtrToInt:
case Instruction::AddrSpaceCast:
// Conservatively return getFull set.
@@ -871,7 +872,8 @@ ConstantRange ConstantRange::signExtend(uint32_t DstTySize) const {
return ConstantRange(Lower.sext(DstTySize), Upper.sext(DstTySize));
}
-ConstantRange ConstantRange::truncate(uint32_t DstTySize) const {
+ConstantRange ConstantRange::truncate(uint32_t DstTySize,
+ unsigned NoWrapKind) const {
assert(getBitWidth() > DstTySize && "Not a value truncation");
if (isEmptySet())
return getEmpty(DstTySize);
@@ -885,22 +887,36 @@ ConstantRange ConstantRange::truncate(uint32_t DstTySize) const {
// We use the non-wrapped set code to analyze the [Lower, MaxValue) part, and
// then we do the union with [MaxValue, Upper)
if (isUpperWrapped()) {
- // If Upper is greater than or equal to MaxValue(DstTy), it covers the whole
- // truncated range.
- if (Upper.getActiveBits() > DstTySize || Upper.countr_one() == DstTySize)
+ // If Upper is greater than MaxValue(DstTy), it covers the whole truncated
+ // range.
+ if (Upper.getActiveBits() > DstTySize)
return getFull(DstTySize);
- Union = ConstantRange(APInt::getMaxValue(DstTySize),Upper.trunc(DstTySize));
- UpperDiv.setAllBits();
-
- // Union covers the MaxValue case, so return if the remaining range is just
- // MaxValue(DstTy).
- if (LowerDiv == UpperDiv)
- return Union;
+ // For nuw the two parts are: [0, Upper) \/ [Lower, MaxValue(DstTy)]
+ if (NoWrapKind & TruncInst::NoUnsignedWrap) {
+ Union = ConstantRange(APInt::getZero(DstTySize), Upper.trunc(DstTySize));
+ UpperDiv = APInt::getOneBitSet(getBitWidth(), DstTySize);
+ } else {
+ // If Upper is equal to MaxValue(DstTy), it covers the whole truncated
+ // range.
+ if (Upper.countr_one() == DstTySize)
+ return getFull(DstTySize);
+ Union =
+ ConstantRange(APInt::getMaxValue(DstTySize), Upper.trunc(DstTySize));
+ UpperDiv.setAllBits();
+ // Union covers the MaxValue case, so return if the remaining range is
+ // just MaxValue(DstTy).
+ if (LowerDiv == UpperDiv)
+ return Union;
+ }
}
// Chop off the most significant bits that are past the destination bitwidth.
if (LowerDiv.getActiveBits() > DstTySize) {
+ // For trunc nuw if LowerDiv is greater than MaxValue(DstTy), the range is
+ // outside the whole truncated range.
+ if (NoWrapKind & TruncInst::NoUnsignedWrap)
+ return Union;
// Mask to just the signficant bits and subtract from LowerDiv/UpperDiv.
APInt Adjust = LowerDiv & APInt::getBitsSetFrom(getBitWidth(), DstTySize);
LowerDiv -= Adjust;
@@ -912,6 +928,10 @@ ConstantRange ConstantRange::truncate(uint32_t DstTySize) const {
return ConstantRange(LowerDiv.trunc(DstTySize),
UpperDiv.trunc(DstTySize)).unionWith(Union);
+ if (!LowerDiv.isZero() && NoWrapKind & TruncInst::NoUnsignedWrap)
+ return ConstantRange(LowerDiv.trunc(DstTySize), APInt::getZero(DstTySize))
+ .unionWith(Union);
+
// The truncated value wraps around. Check if we can do better than fullset.
if (UpperDivWidth == DstTySize + 1) {
// Clear the MSB so that UpperDiv wraps around.
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
index a3c725b..c7e3113a 100644
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -1567,6 +1567,7 @@ Constant *ConstantExpr::getWithOperands(ArrayRef<Constant *> Ops, Type *Ty,
case Instruction::SIToFP:
case Instruction::FPToUI:
case Instruction::FPToSI:
+ case Instruction::PtrToAddr:
case Instruction::PtrToInt:
case Instruction::IntToPtr:
case Instruction::BitCast:
@@ -2223,6 +2224,8 @@ Constant *ConstantExpr::getCast(unsigned oc, Constant *C, Type *Ty,
llvm_unreachable("Invalid cast opcode");
case Instruction::Trunc:
return getTrunc(C, Ty, OnlyIfReduced);
+ case Instruction::PtrToAddr:
+ return getPtrToAddr(C, Ty, OnlyIfReduced);
case Instruction::PtrToInt:
return getPtrToInt(C, Ty, OnlyIfReduced);
case Instruction::IntToPtr:
@@ -2280,6 +2283,20 @@ Constant *ConstantExpr::getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced) {
return getFoldedCast(Instruction::Trunc, C, Ty, OnlyIfReduced);
}
+Constant *ConstantExpr::getPtrToAddr(Constant *C, Type *DstTy,
+ bool OnlyIfReduced) {
+ assert(C->getType()->isPtrOrPtrVectorTy() &&
+ "PtrToAddr source must be pointer or pointer vector");
+ assert(DstTy->isIntOrIntVectorTy() &&
+ "PtrToAddr destination must be integer or integer vector");
+ assert(isa<VectorType>(C->getType()) == isa<VectorType>(DstTy));
+ if (isa<VectorType>(C->getType()))
+ assert(cast<VectorType>(C->getType())->getElementCount() ==
+ cast<VectorType>(DstTy)->getElementCount() &&
+ "Invalid cast between a different number of vector elements");
+ return getFoldedCast(Instruction::PtrToAddr, C, DstTy, OnlyIfReduced);
+}
+
Constant *ConstantExpr::getPtrToInt(Constant *C, Type *DstTy,
bool OnlyIfReduced) {
assert(C->getType()->isPtrOrPtrVectorTy() &&
@@ -2435,6 +2452,7 @@ bool ConstantExpr::isDesirableCastOp(unsigned Opcode) {
case Instruction::FPToSI:
return false;
case Instruction::Trunc:
+ case Instruction::PtrToAddr:
case Instruction::PtrToInt:
case Instruction::IntToPtr:
case Instruction::BitCast:
@@ -2457,6 +2475,7 @@ bool ConstantExpr::isSupportedCastOp(unsigned Opcode) {
case Instruction::FPToSI:
return false;
case Instruction::Trunc:
+ case Instruction::PtrToAddr:
case Instruction::PtrToInt:
case Instruction::IntToPtr:
case Instruction::BitCast:
@@ -3401,6 +3420,7 @@ Instruction *ConstantExpr::getAsInstruction() const {
switch (getOpcode()) {
case Instruction::Trunc:
+ case Instruction::PtrToAddr:
case Instruction::PtrToInt:
case Instruction::IntToPtr:
case Instruction::BitCast:
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index f7ef4aa..8b5965b 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -2186,6 +2186,11 @@ void LLVMGlobalSetMetadata(LLVMValueRef Global, unsigned Kind,
unwrap<GlobalObject>(Global)->setMetadata(Kind, unwrap<MDNode>(MD));
}
+void LLVMGlobalAddMetadata(LLVMValueRef Global, unsigned Kind,
+ LLVMMetadataRef MD) {
+ unwrap<GlobalObject>(Global)->addMetadata(Kind, *unwrap<MDNode>(MD));
+}
+
void LLVMGlobalEraseMetadata(LLVMValueRef Global, unsigned Kind) {
unwrap<GlobalObject>(Global)->eraseMetadata(Kind);
}
@@ -2194,6 +2199,11 @@ void LLVMGlobalClearMetadata(LLVMValueRef Global) {
unwrap<GlobalObject>(Global)->clearMetadata();
}
+void LLVMGlobalAddDebugInfo(LLVMValueRef Global, LLVMMetadataRef GVE) {
+ unwrap<GlobalVariable>(Global)->addDebugInfo(
+ unwrap<DIGlobalVariableExpression>(GVE));
+}
+
/*--.. Operations on global variables ......................................--*/
LLVMValueRef LLVMAddGlobal(LLVMModuleRef M, LLVMTypeRef Ty, const char *Name) {
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index dbd6d81..2cf96f8 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -694,7 +694,12 @@ void DataLayout::setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth,
Align DataLayout::getIntegerAlignment(uint32_t BitWidth,
bool abi_or_pref) const {
- auto I = lower_bound(IntSpecs, BitWidth, LessPrimitiveBitWidth());
+ auto I = IntSpecs.begin();
+ for (; I != IntSpecs.end(); ++I) {
+ if (I->BitWidth >= BitWidth)
+ break;
+ }
+
// If we don't have an exact match, use alignment of next larger integer
// type. If there is none, use alignment of largest integer type by going
// back one element.
@@ -839,6 +844,44 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
}
}
+TypeSize DataLayout::getTypeAllocSize(Type *Ty) const {
+ switch (Ty->getTypeID()) {
+ case Type::ArrayTyID: {
+ // The alignment of the array is the alignment of the element, so there
+ // is no need for further adjustment.
+ auto *ATy = cast<ArrayType>(Ty);
+ return ATy->getNumElements() * getTypeAllocSize(ATy->getElementType());
+ }
+ case Type::StructTyID: {
+ const StructLayout *Layout = getStructLayout(cast<StructType>(Ty));
+ TypeSize Size = Layout->getSizeInBytes();
+
+ if (cast<StructType>(Ty)->isPacked())
+ return Size;
+
+ Align A = std::max(StructABIAlignment, Layout->getAlignment());
+ return alignTo(Size, A.value());
+ }
+ case Type::IntegerTyID: {
+ unsigned BitWidth = Ty->getIntegerBitWidth();
+ TypeSize Size = TypeSize::getFixed(divideCeil(BitWidth, 8));
+ Align A = getIntegerAlignment(BitWidth, /*ABI=*/true);
+ return alignTo(Size, A.value());
+ }
+ case Type::PointerTyID: {
+ unsigned AS = Ty->getPointerAddressSpace();
+ TypeSize Size = TypeSize::getFixed(getPointerSize(AS));
+ return alignTo(Size, getPointerABIAlignment(AS).value());
+ }
+ case Type::TargetExtTyID: {
+ Type *LayoutTy = cast<TargetExtType>(Ty)->getLayoutType();
+ return getTypeAllocSize(LayoutTy);
+ }
+ default:
+ return alignTo(getTypeStoreSize(Ty), getABITypeAlign(Ty).value());
+ }
+}
+
Align DataLayout::getABITypeAlign(Type *Ty) const {
return getAlignment(Ty, true);
}
@@ -926,12 +969,13 @@ static APInt getElementIndex(TypeSize ElemSize, APInt &Offset) {
return APInt::getZero(BitWidth);
}
- APInt Index = Offset.sdiv(ElemSize);
- Offset -= Index * ElemSize;
+ uint64_t FixedElemSize = ElemSize.getFixedValue();
+ APInt Index = Offset.sdiv(FixedElemSize);
+ Offset -= Index * FixedElemSize;
if (Offset.isNegative()) {
// Prefer a positive remaining offset to allow struct indexing.
--Index;
- Offset += ElemSize;
+ Offset += FixedElemSize;
assert(Offset.isNonNegative() && "Remaining offset shouldn't be negative");
}
return Index;
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index ab8ecee..b468d92 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -755,7 +755,7 @@ private:
return getReplacementMDNode(N);
};
- // Seperate recursive doRemap and operator [] into 2 lines to avoid
+ // Separate recursive doRemap and operator [] into 2 lines to avoid
// out-of-order evaluations since both of them can access the same memory
// location in map Replacements.
auto Value = doRemap(N);
@@ -1896,29 +1896,8 @@ AssignmentInstRange at::getAssignmentInsts(DIAssignID *ID) {
return make_range(MapIt->second.begin(), MapIt->second.end());
}
-AssignmentMarkerRange at::getAssignmentMarkers(DIAssignID *ID) {
- assert(ID && "Expected non-null ID");
- LLVMContext &Ctx = ID->getContext();
-
- auto *IDAsValue = MetadataAsValue::getIfExists(Ctx, ID);
-
- // The ID is only used wrapped in MetadataAsValue(ID), so lets check that
- // one of those already exists first.
- if (!IDAsValue)
- return make_range(Value::user_iterator(), Value::user_iterator());
-
- return make_range(IDAsValue->user_begin(), IDAsValue->user_end());
-}
-
void at::deleteAssignmentMarkers(const Instruction *Inst) {
- auto Range = getAssignmentMarkers(Inst);
- SmallVector<DbgVariableRecord *> DVRAssigns = getDVRAssignmentMarkers(Inst);
- if (Range.empty() && DVRAssigns.empty())
- return;
- SmallVector<DbgAssignIntrinsic *> ToDelete(Range.begin(), Range.end());
- for (auto *DAI : ToDelete)
- DAI->eraseFromParent();
- for (auto *DVR : DVRAssigns)
+ for (auto *DVR : getDVRAssignmentMarkers(Inst))
DVR->eraseFromParent();
}
@@ -1936,31 +1915,21 @@ void at::RAUW(DIAssignID *Old, DIAssignID *New) {
}
void at::deleteAll(Function *F) {
- SmallVector<DbgAssignIntrinsic *, 12> ToDelete;
- SmallVector<DbgVariableRecord *, 12> DPToDelete;
for (BasicBlock &BB : *F) {
for (Instruction &I : BB) {
- for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
+ for (DbgVariableRecord &DVR :
+ make_early_inc_range(filterDbgVars(I.getDbgRecordRange())))
if (DVR.isDbgAssign())
- DPToDelete.push_back(&DVR);
- if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(&I))
- ToDelete.push_back(DAI);
- else
- I.setMetadata(LLVMContext::MD_DIAssignID, nullptr);
+ DVR.eraseFromParent();
+
+ I.setMetadata(LLVMContext::MD_DIAssignID, nullptr);
}
}
- for (auto *DAI : ToDelete)
- DAI->eraseFromParent();
- for (auto *DVR : DPToDelete)
- DVR->eraseFromParent();
}
-/// FIXME: Remove this wrapper function and call
-/// DIExpression::calculateFragmentIntersect directly.
-template <typename T>
-bool calculateFragmentIntersectImpl(
+bool at::calculateFragmentIntersect(
const DataLayout &DL, const Value *Dest, uint64_t SliceOffsetInBits,
- uint64_t SliceSizeInBits, const T *AssignRecord,
+ uint64_t SliceSizeInBits, const DbgVariableRecord *AssignRecord,
std::optional<DIExpression::FragmentInfo> &Result) {
// No overlap if this DbgRecord describes a killed location.
if (AssignRecord->isKillAddress())
@@ -1989,26 +1958,6 @@ bool calculateFragmentIntersectImpl(
BitExtractOffsetInBits, VarFrag, Result, OffsetFromLocationInBits);
}
-/// FIXME: Remove this wrapper function and call
-/// DIExpression::calculateFragmentIntersect directly.
-bool at::calculateFragmentIntersect(
- const DataLayout &DL, const Value *Dest, uint64_t SliceOffsetInBits,
- uint64_t SliceSizeInBits, const DbgAssignIntrinsic *DbgAssign,
- std::optional<DIExpression::FragmentInfo> &Result) {
- return calculateFragmentIntersectImpl(DL, Dest, SliceOffsetInBits,
- SliceSizeInBits, DbgAssign, Result);
-}
-
-/// FIXME: Remove this wrapper function and call
-/// DIExpression::calculateFragmentIntersect directly.
-bool at::calculateFragmentIntersect(
- const DataLayout &DL, const Value *Dest, uint64_t SliceOffsetInBits,
- uint64_t SliceSizeInBits, const DbgVariableRecord *DVRAssign,
- std::optional<DIExpression::FragmentInfo> &Result) {
- return calculateFragmentIntersectImpl(DL, Dest, SliceOffsetInBits,
- SliceSizeInBits, DVRAssign, Result);
-}
-
/// Update inlined instructions' DIAssignID metadata. We need to do this
/// otherwise a function inlined more than once into the same function
/// will cause DIAssignID to be shared by many instructions.
@@ -2029,8 +1978,6 @@ void at::remapAssignID(DenseMap<DIAssignID *, DIAssignID *> &Map,
}
if (auto *ID = I.getMetadata(LLVMContext::MD_DIAssignID))
I.setMetadata(LLVMContext::MD_DIAssignID, GetNewID(ID));
- else if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(&I))
- DAI->setAssignId(GetNewID(DAI->getAssignID()));
}
/// Collect constant properies (base, size, offset) of \p StoreDest.
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index f1d4549..96065ed 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -57,15 +57,9 @@ DebugVariable::DebugVariable(const DbgVariableRecord *DVR)
DILocation::DILocation(LLVMContext &C, StorageType Storage, unsigned Line,
unsigned Column, uint64_t AtomGroup, uint8_t AtomRank,
ArrayRef<Metadata *> MDs, bool ImplicitCode)
- : MDNode(C, DILocationKind, Storage, MDs)
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
- ,
- AtomGroup(AtomGroup), AtomRank(AtomRank)
-#endif
-{
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
+ : MDNode(C, DILocationKind, Storage, MDs), AtomGroup(AtomGroup),
+ AtomRank(AtomRank) {
assert(AtomRank <= 7 && "AtomRank number should fit in 3 bits");
-#endif
if (AtomGroup)
C.updateDILocationAtomGroupWaterline(AtomGroup + 1);
diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp
index 7b799c7..11d33e2 100644
--- a/llvm/lib/IR/Globals.cpp
+++ b/llvm/lib/IR/Globals.cpp
@@ -404,6 +404,7 @@ findBaseObject(const Constant *C, DenseSet<const GlobalAlias *> &Aliases,
return findBaseObject(CE->getOperand(0), Aliases, Op);
}
case Instruction::IntToPtr:
+ case Instruction::PtrToAddr:
case Instruction::PtrToInt:
case Instruction::BitCast:
case Instruction::GetElementPtr:
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 49c6dc7..614c3a9 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -411,28 +411,16 @@ CallInst *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_fminimum, Src);
}
-CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
+CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr) {
assert(isa<PointerType>(Ptr->getType()) &&
"lifetime.start only applies to pointers.");
- if (!Size)
- Size = getInt64(-1);
- else
- assert(Size->getType() == getInt64Ty() &&
- "lifetime.start requires the size to be an i64");
- Value *Ops[] = { Size, Ptr };
- return CreateIntrinsic(Intrinsic::lifetime_start, {Ptr->getType()}, Ops);
+ return CreateIntrinsic(Intrinsic::lifetime_start, {Ptr->getType()}, {Ptr});
}
-CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
+CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr) {
assert(isa<PointerType>(Ptr->getType()) &&
"lifetime.end only applies to pointers.");
- if (!Size)
- Size = getInt64(-1);
- else
- assert(Size->getType() == getInt64Ty() &&
- "lifetime.end requires the size to be an i64");
- Value *Ops[] = { Size, Ptr };
- return CreateIntrinsic(Intrinsic::lifetime_end, {Ptr->getType()}, Ops);
+ return CreateIntrinsic(Intrinsic::lifetime_end, {Ptr->getType()}, {Ptr});
}
CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 763cc18..5e87b5f 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -26,9 +26,18 @@
#include "llvm/IR/Operator.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Type.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
using namespace llvm;
+// FIXME: Flag used for an ablation performance test, Issue #147390. Placing it
+// here because referencing IR should be feasible from anywhere. Will be
+// removed after the ablation test.
+cl::opt<bool> ProfcheckDisableMetadataFixes(
+ "profcheck-disable-metadata-fixes", cl::Hidden, cl::init(false),
+ cl::desc(
+ "Disable metadata propagation fixes discovered through Issue #147390"));
+
InsertPosition::InsertPosition(Instruction *InsertBefore)
: InsertAt(InsertBefore ? InsertBefore->getIterator()
: InstListType::iterator()) {}
@@ -543,14 +552,19 @@ void Instruction::dropUBImplyingAttrsAndUnknownMetadata(
CB->removeRetAttrs(UBImplyingAttributes);
}
-void Instruction::dropUBImplyingAttrsAndMetadata() {
+void Instruction::dropUBImplyingAttrsAndMetadata(ArrayRef<unsigned> Keep) {
// !annotation metadata does not impact semantics.
// !range, !nonnull and !align produce poison, so they are safe to speculate.
// !noundef and various AA metadata must be dropped, as it generally produces
// immediate undefined behavior.
- unsigned KnownIDs[] = {LLVMContext::MD_annotation, LLVMContext::MD_range,
- LLVMContext::MD_nonnull, LLVMContext::MD_align};
- dropUBImplyingAttrsAndUnknownMetadata(KnownIDs);
+ static const unsigned KnownIDs[] = {
+ LLVMContext::MD_annotation, LLVMContext::MD_range,
+ LLVMContext::MD_nonnull, LLVMContext::MD_align};
+ SmallVector<unsigned> KeepIDs;
+ KeepIDs.reserve(Keep.size() + std::size(KnownIDs));
+ append_range(KeepIDs, KnownIDs);
+ append_range(KeepIDs, Keep);
+ dropUBImplyingAttrsAndUnknownMetadata(KeepIDs);
}
bool Instruction::hasUBImplyingAttrs() const {
@@ -817,6 +831,7 @@ const char *Instruction::getOpcodeName(unsigned OpCode) {
case UIToFP: return "uitofp";
case SIToFP: return "sitofp";
case IntToPtr: return "inttoptr";
+ case PtrToAddr: return "ptrtoaddr";
case PtrToInt: return "ptrtoint";
case BitCast: return "bitcast";
case AddrSpaceCast: return "addrspacecast";
@@ -942,14 +957,13 @@ bool Instruction::isIdenticalToWhenDefined(const Instruction *I,
// We have two instructions of identical opcode and #operands. Check to see
// if all operands are the same.
- if (!std::equal(op_begin(), op_end(), I->op_begin()))
+ if (!equal(operands(), I->operands()))
return false;
// WARNING: this logic must be kept in sync with EliminateDuplicatePHINodes()!
- if (const PHINode *thisPHI = dyn_cast<PHINode>(this)) {
- const PHINode *otherPHI = cast<PHINode>(I);
- return std::equal(thisPHI->block_begin(), thisPHI->block_end(),
- otherPHI->block_begin());
+ if (const PHINode *Phi = dyn_cast<PHINode>(this)) {
+ const PHINode *OtherPhi = cast<PHINode>(I);
+ return equal(Phi->blocks(), OtherPhi->blocks());
}
return this->hasSameSpecialState(I, /*IgnoreAlignment=*/false,
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index b896382..a1751c0 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -2798,6 +2798,7 @@ bool CastInst::isNoopCast(Instruction::CastOps Opcode,
return false;
case Instruction::BitCast:
return true; // BitCast never modifies bits.
+ case Instruction::PtrToAddr:
case Instruction::PtrToInt:
return DL.getIntPtrType(SrcTy)->getScalarSizeInBits() ==
DestTy->getScalarSizeInBits();
@@ -2855,26 +2856,29 @@ unsigned CastInst::isEliminableCastPair(
// same reason.
const unsigned numCastOps =
Instruction::CastOpsEnd - Instruction::CastOpsBegin;
+ // clang-format off
static const uint8_t CastResults[numCastOps][numCastOps] = {
- // T F F U S F F P I B A -+
- // R Z S P P I I T P 2 N T S |
- // U E E 2 2 2 2 R E I T C C +- secondOp
- // N X X U S F F N X N 2 V V |
- // C T T I I P P C T T P T T -+
- { 1, 0, 0,99,99, 0, 0,99,99,99, 0, 3, 0}, // Trunc -+
- { 8, 1, 9,99,99, 2,17,99,99,99, 2, 3, 0}, // ZExt |
- { 8, 0, 1,99,99, 0, 2,99,99,99, 0, 3, 0}, // SExt |
- { 0, 0, 0,99,99, 0, 0,99,99,99, 0, 3, 0}, // FPToUI |
- { 0, 0, 0,99,99, 0, 0,99,99,99, 0, 3, 0}, // FPToSI |
- { 99,99,99, 0, 0,99,99, 0, 0,99,99, 4, 0}, // UIToFP +- firstOp
- { 99,99,99, 0, 0,99,99, 0, 0,99,99, 4, 0}, // SIToFP |
- { 99,99,99, 0, 0,99,99, 0, 0,99,99, 4, 0}, // FPTrunc |
- { 99,99,99, 2, 2,99,99, 8, 2,99,99, 4, 0}, // FPExt |
- { 1, 0, 0,99,99, 0, 0,99,99,99, 7, 3, 0}, // PtrToInt |
- { 99,99,99,99,99,99,99,99,99,11,99,15, 0}, // IntToPtr |
- { 5, 5, 5, 0, 0, 5, 5, 0, 0,16, 5, 1,14}, // BitCast |
- { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,13,12}, // AddrSpaceCast -+
+ // T F F U S F F P P I B A -+
+ // R Z S P P I I T P 2 2 N T S |
+ // U E E 2 2 2 2 R E I A T C C +- secondOp
+ // N X X U S F F N X N D 2 V V |
+ // C T T I I P P C T T R P T T -+
+ { 1, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // Trunc -+
+ { 8, 1, 9,99,99, 2,17,99,99,99,99, 2, 3, 0}, // ZExt |
+ { 8, 0, 1,99,99, 0, 2,99,99,99,99, 0, 3, 0}, // SExt |
+ { 0, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // FPToUI |
+ { 0, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // FPToSI |
+ { 99,99,99, 0, 0,99,99, 0, 0,99,99,99, 4, 0}, // UIToFP +- firstOp
+ { 99,99,99, 0, 0,99,99, 0, 0,99,99,99, 4, 0}, // SIToFP |
+ { 99,99,99, 0, 0,99,99, 0, 0,99,99,99, 4, 0}, // FPTrunc |
+ { 99,99,99, 2, 2,99,99, 8, 2,99,99,99, 4, 0}, // FPExt |
+ { 1, 0, 0,99,99, 0, 0,99,99,99,99, 7, 3, 0}, // PtrToInt |
+ { 1, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // PtrToAddr |
+ { 99,99,99,99,99,99,99,99,99,11,99,99,15, 0}, // IntToPtr |
+ { 5, 5, 5, 0, 0, 5, 5, 0, 0,16,16, 5, 1,14}, // BitCast |
+ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,13,12}, // AddrSpaceCast -+
};
+ // clang-format on
// TODO: This logic could be encoded into the table above and handled in the
// switch below.
@@ -3046,6 +3050,7 @@ CastInst *CastInst::Create(Instruction::CastOps op, Value *S, Type *Ty,
case SIToFP: return new SIToFPInst (S, Ty, Name, InsertBefore);
case FPToUI: return new FPToUIInst (S, Ty, Name, InsertBefore);
case FPToSI: return new FPToSIInst (S, Ty, Name, InsertBefore);
+ case PtrToAddr: return new PtrToAddrInst (S, Ty, Name, InsertBefore);
case PtrToInt: return new PtrToIntInst (S, Ty, Name, InsertBefore);
case IntToPtr: return new IntToPtrInst (S, Ty, Name, InsertBefore);
case BitCast:
@@ -3347,6 +3352,7 @@ CastInst::castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy) {
case Instruction::FPToSI:
return SrcTy->isFPOrFPVectorTy() && DstTy->isIntOrIntVectorTy() &&
SrcEC == DstEC;
+ case Instruction::PtrToAddr:
case Instruction::PtrToInt:
if (SrcEC != DstEC)
return false;
@@ -3460,6 +3466,12 @@ PtrToIntInst::PtrToIntInst(Value *S, Type *Ty, const Twine &Name,
assert(castIsValid(getOpcode(), S, Ty) && "Illegal PtrToInt");
}
+PtrToAddrInst::PtrToAddrInst(Value *S, Type *Ty, const Twine &Name,
+ InsertPosition InsertBefore)
+ : CastInst(Ty, PtrToAddr, S, Name, InsertBefore) {
+ assert(castIsValid(getOpcode(), S, Ty) && "Illegal PtrToAddr");
+}
+
IntToPtrInst::IntToPtrInst(Value *S, Type *Ty, const Twine &Name,
InsertPosition InsertBefore)
: CastInst(Ty, IntToPtr, S, Name, InsertBefore) {
@@ -4427,6 +4439,10 @@ PtrToIntInst *PtrToIntInst::cloneImpl() const {
return new PtrToIntInst(getOperand(0), getType());
}
+PtrToAddrInst *PtrToAddrInst::cloneImpl() const {
+ return new PtrToAddrInst(getOperand(0), getType());
+}
+
IntToPtrInst *IntToPtrInst::cloneImpl() const {
return new IntToPtrInst(getOperand(0), getType());
}
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index b1d3339..23a4d1b 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -448,6 +448,7 @@ VPIntrinsic::getMemoryPointerParamPos(Intrinsic::ID VPID) {
case Intrinsic::experimental_vp_strided_store:
return 1;
case Intrinsic::vp_load:
+ case Intrinsic::vp_load_ff:
case Intrinsic::vp_gather:
case Intrinsic::experimental_vp_strided_load:
return 0;
@@ -671,6 +672,10 @@ Function *VPIntrinsic::getOrInsertDeclarationForParams(
VPFunc = Intrinsic::getOrInsertDeclaration(
M, VPID, {ReturnType, Params[0]->getType()});
break;
+ case Intrinsic::vp_load_ff:
+ VPFunc = Intrinsic::getOrInsertDeclaration(
+ M, VPID, {ReturnType->getStructElementType(0), Params[0]->getType()});
+ break;
case Intrinsic::experimental_vp_strided_load:
VPFunc = Intrinsic::getOrInsertDeclaration(
M, VPID, {ReturnType, Params[0]->getType(), Params[1]->getType()});
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index aa2a60e..e03f993 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -312,10 +312,8 @@ template <> struct MDNodeKeyImpl<MDTuple> : MDNodeOpsKey {
template <> struct MDNodeKeyImpl<DILocation> {
Metadata *Scope;
Metadata *InlinedAt;
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
uint64_t AtomGroup : 61;
uint64_t AtomRank : 3;
-#endif
unsigned Line;
uint16_t Column;
bool ImplicitCode;
@@ -323,36 +321,24 @@ template <> struct MDNodeKeyImpl<DILocation> {
MDNodeKeyImpl(unsigned Line, uint16_t Column, Metadata *Scope,
Metadata *InlinedAt, bool ImplicitCode, uint64_t AtomGroup,
uint8_t AtomRank)
- : Scope(Scope), InlinedAt(InlinedAt),
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
- AtomGroup(AtomGroup), AtomRank(AtomRank),
-#endif
- Line(Line), Column(Column), ImplicitCode(ImplicitCode) {
- }
+ : Scope(Scope), InlinedAt(InlinedAt), AtomGroup(AtomGroup),
+ AtomRank(AtomRank), Line(Line), Column(Column),
+ ImplicitCode(ImplicitCode) {}
MDNodeKeyImpl(const DILocation *L)
: Scope(L->getRawScope()), InlinedAt(L->getRawInlinedAt()),
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
AtomGroup(L->getAtomGroup()), AtomRank(L->getAtomRank()),
-#endif
Line(L->getLine()), Column(L->getColumn()),
- ImplicitCode(L->isImplicitCode()) {
- }
+ ImplicitCode(L->isImplicitCode()) {}
bool isKeyOf(const DILocation *RHS) const {
return Line == RHS->getLine() && Column == RHS->getColumn() &&
Scope == RHS->getRawScope() && InlinedAt == RHS->getRawInlinedAt() &&
- ImplicitCode == RHS->isImplicitCode()
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
- && AtomGroup == RHS->getAtomGroup() &&
- AtomRank == RHS->getAtomRank();
-#else
- ;
-#endif
+ ImplicitCode == RHS->isImplicitCode() &&
+ AtomGroup == RHS->getAtomGroup() && AtomRank == RHS->getAtomRank();
}
unsigned getHashValue() const {
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
// Hashing AtomGroup and AtomRank substantially impacts performance whether
// Key Instructions is enabled or not. We can't detect whether it's enabled
// here cheaply; avoiding hashing zero values is a good approximation. This
@@ -363,7 +349,6 @@ template <> struct MDNodeKeyImpl<DILocation> {
if (AtomGroup || AtomRank)
return hash_combine(Line, Column, Scope, InlinedAt, ImplicitCode,
AtomGroup, (uint8_t)AtomRank);
-#endif
return hash_combine(Line, Column, Scope, InlinedAt, ImplicitCode);
}
};
diff --git a/llvm/lib/IR/Mangler.cpp b/llvm/lib/IR/Mangler.cpp
index 010bd15..ca6a480 100644
--- a/llvm/lib/IR/Mangler.cpp
+++ b/llvm/lib/IR/Mangler.cpp
@@ -292,6 +292,9 @@ void llvm::emitLinkerFlagsForUsedCOFF(raw_ostream &OS, const GlobalValue *GV,
}
std::optional<std::string> llvm::getArm64ECMangledFunctionName(StringRef Name) {
+ assert(!Name.empty() &&
+ "getArm64ECMangledFunctionName requires non-empty name");
+
if (Name[0] != '?') {
// For non-C++ symbols, prefix the name with "#" unless it's already
// mangled.
diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp
index b1b5f67..d24263f 100644
--- a/llvm/lib/IR/ProfDataUtils.cpp
+++ b/llvm/lib/IR/ProfDataUtils.cpp
@@ -270,6 +270,18 @@ void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
I.setMetadata(LLVMContext::MD_prof, BranchWeights);
}
+SmallVector<uint32_t> downscaleWeights(ArrayRef<uint64_t> Weights,
+ std::optional<uint64_t> KnownMaxCount) {
+ uint64_t MaxCount = KnownMaxCount.has_value() ? KnownMaxCount.value()
+ : *llvm::max_element(Weights);
+ assert(MaxCount > 0 && "Bad max count");
+ uint64_t Scale = calculateCountScale(MaxCount);
+ SmallVector<uint32_t> DownscaledWeights;
+ for (const auto &ECI : Weights)
+ DownscaledWeights.push_back(scaleBranchCount(ECI, Scale));
+ return DownscaledWeights;
+}
+
void scaleProfData(Instruction &I, uint64_t S, uint64_t T) {
assert(T != 0 && "Caller should guarantee");
auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index 1ca5878..4fe5714 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -8,65 +8,22 @@
#include "llvm/IR/RuntimeLibcalls.h"
#include "llvm/ADT/StringTable.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/xxhash.h"
+#include "llvm/TargetParser/ARMTargetParser.h"
+
+#define DEBUG_TYPE "runtime-libcalls-info"
using namespace llvm;
using namespace RTLIB;
#define GET_INIT_RUNTIME_LIBCALL_NAMES
#define GET_SET_TARGET_RUNTIME_LIBCALL_SETS
+#define DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME
#include "llvm/IR/RuntimeLibcalls.inc"
#undef GET_INIT_RUNTIME_LIBCALL_NAMES
#undef GET_SET_TARGET_RUNTIME_LIBCALL_SETS
-
-static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT,
- FloatABI::ABIType FloatABIType,
- EABI EABIVersion) {
- static const RTLIB::LibcallImpl AAPCS_Libcalls[] = {
- RTLIB::__aeabi_dadd, RTLIB::__aeabi_ddiv,
- RTLIB::__aeabi_dmul, RTLIB::__aeabi_dsub,
- RTLIB::__aeabi_dcmpeq__oeq, RTLIB::__aeabi_dcmpeq__une,
- RTLIB::__aeabi_dcmplt, RTLIB::__aeabi_dcmple,
- RTLIB::__aeabi_dcmpge, RTLIB::__aeabi_dcmpgt,
- RTLIB::__aeabi_dcmpun, RTLIB::__aeabi_fadd,
- RTLIB::__aeabi_fdiv, RTLIB::__aeabi_fmul,
- RTLIB::__aeabi_fsub, RTLIB::__aeabi_fcmpeq__oeq,
- RTLIB::__aeabi_fcmpeq__une, RTLIB::__aeabi_fcmplt,
- RTLIB::__aeabi_fcmple, RTLIB::__aeabi_fcmpge,
- RTLIB::__aeabi_fcmpgt, RTLIB::__aeabi_fcmpun,
- RTLIB::__aeabi_d2iz, RTLIB::__aeabi_d2uiz,
- RTLIB::__aeabi_d2lz, RTLIB::__aeabi_d2ulz,
- RTLIB::__aeabi_f2iz, RTLIB::__aeabi_f2uiz,
- RTLIB::__aeabi_f2lz, RTLIB::__aeabi_f2ulz,
- RTLIB::__aeabi_d2f, RTLIB::__aeabi_d2h,
- RTLIB::__aeabi_f2d, RTLIB::__aeabi_i2d,
- RTLIB::__aeabi_ui2d, RTLIB::__aeabi_l2d,
- RTLIB::__aeabi_ul2d, RTLIB::__aeabi_i2f,
- RTLIB::__aeabi_ui2f, RTLIB::__aeabi_l2f,
- RTLIB::__aeabi_ul2f, RTLIB::__aeabi_lmul,
- RTLIB::__aeabi_llsl, RTLIB::__aeabi_llsr,
- RTLIB::__aeabi_lasr, RTLIB::__aeabi_idiv__i8,
- RTLIB::__aeabi_idiv__i16, RTLIB::__aeabi_idiv__i32,
- RTLIB::__aeabi_idivmod, RTLIB::__aeabi_uidivmod,
- RTLIB::__aeabi_ldivmod, RTLIB::__aeabi_uidiv__i8,
- RTLIB::__aeabi_uidiv__i16, RTLIB::__aeabi_uidiv__i32,
- RTLIB::__aeabi_uldivmod, RTLIB::__aeabi_f2h,
- RTLIB::__aeabi_d2h, RTLIB::__aeabi_h2f,
- RTLIB::__aeabi_memcpy, RTLIB::__aeabi_memmove,
- RTLIB::__aeabi_memset, RTLIB::__aeabi_memcpy4,
- RTLIB::__aeabi_memcpy8, RTLIB::__aeabi_memmove4,
- RTLIB::__aeabi_memmove8, RTLIB::__aeabi_memset4,
- RTLIB::__aeabi_memset8, RTLIB::__aeabi_memclr,
- RTLIB::__aeabi_memclr4, RTLIB::__aeabi_memclr8};
-
- for (RTLIB::LibcallImpl Impl : AAPCS_Libcalls)
- Info.setLibcallImplCallingConv(Impl, CallingConv::ARM_AAPCS);
-}
-
-void RTLIB::RuntimeLibcallsInfo::initDefaultLibCallImpls() {
- std::memcpy(LibcallImpls, DefaultLibcallImpls, sizeof(LibcallImpls));
- static_assert(sizeof(LibcallImpls) == sizeof(DefaultLibcallImpls),
- "libcall array size should match");
-}
+#undef DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME
/// Set default libcall names. If a target wants to opt-out of a libcall it
/// should be placed here.
@@ -74,114 +31,57 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
ExceptionHandling ExceptionModel,
FloatABI::ABIType FloatABI,
EABI EABIVersion, StringRef ABIName) {
- setTargetRuntimeLibcallSets(TT, FloatABI);
-
- // Early exit for targets that have fully ported to tablegen.
- if (TT.isAMDGPU() || TT.isNVPTX() || TT.isWasm())
- return;
+ setTargetRuntimeLibcallSets(TT, ExceptionModel, FloatABI, EABIVersion,
+ ABIName);
- if (TT.isX86() || TT.isVE() || TT.isARM() || TT.isThumb()) {
- if (ExceptionModel == ExceptionHandling::SjLj)
- setLibcallImpl(RTLIB::UNWIND_RESUME, RTLIB::_Unwind_SjLj_Resume);
- }
-
- // A few names are different on particular architectures or environments.
- if (TT.isOSDarwin()) {
- // For f16/f32 conversions, Darwin uses the standard naming scheme,
- // instead of the gnueabi-style __gnu_*_ieee.
- // FIXME: What about other targets?
- setLibcallImpl(RTLIB::FPEXT_F16_F32, RTLIB::__extendhfsf2);
- setLibcallImpl(RTLIB::FPROUND_F32_F16, RTLIB::__truncsfhf2);
-
- if (!darwinHasExp10(TT)) {
- setLibcallImpl(RTLIB::EXP10_F32, RTLIB::Unsupported);
- setLibcallImpl(RTLIB::EXP10_F64, RTLIB::Unsupported);
+ if (TT.isARM() || TT.isThumb()) {
+ // The half <-> float conversion functions are always soft-float on
+ // non-watchos platforms, but are needed for some targets which use a
+ // hard-float calling convention by default.
+ if (!TT.isWatchABI()) {
+ if (isAAPCS_ABI(TT, ABIName)) {
+ setLibcallImplCallingConv(RTLIB::impl___truncsfhf2,
+ CallingConv::ARM_AAPCS);
+ setLibcallImplCallingConv(RTLIB::impl___truncdfhf2,
+ CallingConv::ARM_AAPCS);
+ setLibcallImplCallingConv(RTLIB::impl___extendhfsf2,
+ CallingConv::ARM_AAPCS);
+ } else {
+ setLibcallImplCallingConv(RTLIB::impl___truncsfhf2,
+ CallingConv::ARM_APCS);
+ setLibcallImplCallingConv(RTLIB::impl___truncdfhf2,
+ CallingConv::ARM_APCS);
+ setLibcallImplCallingConv(RTLIB::impl___extendhfsf2,
+ CallingConv::ARM_APCS);
+ }
}
- }
- if (TT.isOSOpenBSD()) {
- setLibcallImpl(RTLIB::STACKPROTECTOR_CHECK_FAIL, RTLIB::Unsupported);
- setLibcallImpl(RTLIB::STACK_SMASH_HANDLER, RTLIB::__stack_smash_handler);
- }
-
- // Skip default manual processing for targets that have been mostly ported to
- // tablegen for now. Eventually the rest of this should be deleted.
- if (TT.isX86() || TT.isAArch64() || TT.isWasm() || TT.isPPC())
return;
-
- if (TT.isARM() || TT.isThumb()) {
- setARMLibcallNames(*this, TT, FloatABI, EABIVersion);
- return;
- }
-
- if (hasSinCos(TT)) {
- setLibcallImpl(RTLIB::SINCOS_F32, RTLIB::sincosf);
- setLibcallImpl(RTLIB::SINCOS_F64, RTLIB::sincos);
- setLibcallImpl(RTLIB::SINCOS_F128, RTLIB::sincos_f128);
- }
-
- setLibcallImpl(RTLIB::EXP10_F32, RTLIB::exp10f);
- setLibcallImpl(RTLIB::EXP10_F64, RTLIB::exp10);
- setLibcallImpl(RTLIB::EXP10_F128, RTLIB::exp10l_f128);
-
- // These libcalls are only available in compiler-rt, not libgcc.
- if (TT.isArch64Bit()) {
- setLibcallImpl(RTLIB::SHL_I128, RTLIB::__ashlti3);
- setLibcallImpl(RTLIB::SRL_I128, RTLIB::__lshrti3);
- setLibcallImpl(RTLIB::SRA_I128, RTLIB::__ashrti3);
- setLibcallImpl(RTLIB::MUL_I128, RTLIB::__multi3);
- setLibcallImpl(RTLIB::MULO_I64, RTLIB::__mulodi4);
- }
-
- if (TT.getArch() == Triple::ArchType::msp430) {
- setLibcallImplCallingConv(RTLIB::__mspabi_mpyll,
- CallingConv::MSP430_BUILTIN);
}
}
-RTLIB::LibcallImpl
-RuntimeLibcallsInfo::getSupportedLibcallImpl(StringRef FuncName) const {
- const ArrayRef<uint16_t> RuntimeLibcallNameOffsets(
- RuntimeLibcallNameOffsetTable);
-
- iterator_range<ArrayRef<uint16_t>::const_iterator> Range =
- getRecognizedLibcallImpls(FuncName);
-
- for (auto I = Range.begin(); I != Range.end(); ++I) {
- RTLIB::LibcallImpl Impl =
- static_cast<RTLIB::LibcallImpl>(I - RuntimeLibcallNameOffsets.begin());
-
- // FIXME: This should not depend on looking up ImplToLibcall, only the list
- // of libcalls for the module.
- RTLIB::LibcallImpl Recognized = LibcallImpls[ImplToLibcall[Impl]];
- if (Recognized != RTLIB::Unsupported)
- return Recognized;
+LLVM_ATTRIBUTE_ALWAYS_INLINE
+iota_range<RTLIB::LibcallImpl>
+RuntimeLibcallsInfo::libcallImplNameHit(uint16_t NameOffsetEntry,
+ uint16_t StrOffset) {
+ int NumAliases = 1;
+ for (uint16_t Entry : ArrayRef(RuntimeLibcallNameOffsetTable)
+ .drop_front(NameOffsetEntry + 1)) {
+ if (Entry != StrOffset)
+ break;
+ ++NumAliases;
}
- return RTLIB::Unsupported;
+ RTLIB::LibcallImpl ImplStart = static_cast<RTLIB::LibcallImpl>(
+ &RuntimeLibcallNameOffsetTable[NameOffsetEntry] -
+ &RuntimeLibcallNameOffsetTable[0]);
+ return enum_seq(ImplStart,
+ static_cast<RTLIB::LibcallImpl>(ImplStart + NumAliases));
}
-iterator_range<ArrayRef<uint16_t>::const_iterator>
-RuntimeLibcallsInfo::getRecognizedLibcallImpls(StringRef FuncName) {
- StringTable::Iterator It = lower_bound(RuntimeLibcallImplNameTable, FuncName);
- if (It == RuntimeLibcallImplNameTable.end() || *It != FuncName)
- return iterator_range(ArrayRef<uint16_t>());
-
- uint16_t IndexVal = It.offset().value();
- const ArrayRef<uint16_t> TableRef(RuntimeLibcallNameOffsetTable);
-
- ArrayRef<uint16_t>::const_iterator E = TableRef.end();
- ArrayRef<uint16_t>::const_iterator EntriesBegin =
- std::lower_bound(TableRef.begin(), E, IndexVal);
- ArrayRef<uint16_t>::const_iterator EntriesEnd = EntriesBegin;
-
- while (EntriesEnd != E && *EntriesEnd == IndexVal)
- ++EntriesEnd;
-
- assert(EntriesBegin != E &&
- "libcall found in name table but not offset table");
-
- return make_range(EntriesBegin, EntriesEnd);
+bool RuntimeLibcallsInfo::isAAPCS_ABI(const Triple &TT, StringRef ABIName) {
+ const ARM::ARMABI TargetABI = ARM::computeTargetABI(TT, ABIName);
+ return TargetABI == ARM::ARM_ABI_AAPCS || TargetABI == ARM::ARM_ABI_AAPCS16;
}
bool RuntimeLibcallsInfo::darwinHasExp10(const Triple &TT) {
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index 9c34662..9db48e8 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -1036,7 +1036,8 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) {
// DirectX resources
if (Name.starts_with("dx."))
return TargetTypeInfo(PointerType::get(C, 0), TargetExtType::CanBeGlobal,
- TargetExtType::CanBeLocal);
+ TargetExtType::CanBeLocal,
+ TargetExtType::IsTokenLike);
// Opaque types in the AMDGPU name space.
if (Name == "amdgcn.named.barrier") {
@@ -1054,6 +1055,14 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) {
return TargetTypeInfo(Type::getVoidTy(C));
}
+bool Type::isTokenLikeTy() const {
+ if (isTokenTy())
+ return true;
+ if (auto *TT = dyn_cast<TargetExtType>(this))
+ return TT->hasProperty(TargetExtType::Property::IsTokenLike);
+ return false;
+}
+
Type *TargetExtType::getLayoutType() const {
return getTargetTypeInfo(this).LayoutType;
}
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index 129ca4a..4e8f359 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -747,34 +747,28 @@ const Value *Value::stripAndAccumulateConstantOffsets(
// means when we construct GEPOffset, we need to use the size
// of GEP's pointer type rather than the size of the original
// pointer type.
- unsigned CurBitWidth = DL.getIndexTypeSizeInBits(V->getType());
- if (CurBitWidth == BitWidth) {
- if (!GEP->accumulateConstantOffset(DL, Offset, ExternalAnalysis))
- return V;
- } else {
- APInt GEPOffset(CurBitWidth, 0);
- if (!GEP->accumulateConstantOffset(DL, GEPOffset, ExternalAnalysis))
- return V;
+ APInt GEPOffset(DL.getIndexTypeSizeInBits(V->getType()), 0);
+ if (!GEP->accumulateConstantOffset(DL, GEPOffset, ExternalAnalysis))
+ return V;
- // Stop traversal if the pointer offset wouldn't fit in the bit-width
- // provided by the Offset argument. This can happen due to AddrSpaceCast
- // stripping.
- if (GEPOffset.getSignificantBits() > BitWidth)
- return V;
+ // Stop traversal if the pointer offset wouldn't fit in the bit-width
+ // provided by the Offset argument. This can happen due to AddrSpaceCast
+ // stripping.
+ if (GEPOffset.getSignificantBits() > BitWidth)
+ return V;
- // External Analysis can return a result higher/lower than the value
- // represents. We need to detect overflow/underflow.
- APInt GEPOffsetST = GEPOffset.sextOrTrunc(BitWidth);
- if (!ExternalAnalysis) {
- Offset += GEPOffsetST;
- } else {
- bool Overflow = false;
- APInt OldOffset = Offset;
- Offset = Offset.sadd_ov(GEPOffsetST, Overflow);
- if (Overflow) {
- Offset = OldOffset;
- return V;
- }
+ // External Analysis can return a result higher/lower than the value
+ // represents. We need to detect overflow/underflow.
+ APInt GEPOffsetST = GEPOffset.sextOrTrunc(BitWidth);
+ if (!ExternalAnalysis) {
+ Offset += GEPOffsetST;
+ } else {
+ bool Overflow = false;
+ APInt OldOffset = Offset;
+ Offset = Offset.sadd_ov(GEPOffsetST, Overflow);
+ if (Overflow) {
+ Offset = OldOffset;
+ return V;
}
}
V = GEP->getPointerOperand();
@@ -842,6 +836,9 @@ bool Value::canBeFreed() const {
return false;
}
+ if (isa<IntToPtrInst>(this) && getMetadata(LLVMContext::MD_nofree))
+ return false;
+
const Function *F = nullptr;
if (auto *I = dyn_cast<Instruction>(this))
F = I->getFunction();
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 3ff9895..b285150 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -55,7 +55,6 @@
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
@@ -527,6 +526,7 @@ private:
void visitRangeMetadata(Instruction &I, MDNode *Range, Type *Ty);
void visitNoaliasAddrspaceMetadata(Instruction &I, MDNode *Range, Type *Ty);
void visitDereferenceableMetadata(Instruction &I, MDNode *MD);
+ void visitNofreeMetadata(Instruction &I, MDNode *MD);
void visitProfMetadata(Instruction &I, MDNode *MD);
void visitCallStackMetadata(MDNode *MD);
void visitMemProfMetadata(Instruction &I, MDNode *MD);
@@ -566,6 +566,8 @@ private:
void visitUIToFPInst(UIToFPInst &I);
void visitSIToFPInst(SIToFPInst &I);
void visitIntToPtrInst(IntToPtrInst &I);
+ void checkPtrToAddr(Type *SrcTy, Type *DestTy, const Value &V);
+ void visitPtrToAddrInst(PtrToAddrInst &I);
void visitPtrToIntInst(PtrToIntInst &I);
void visitBitCastInst(BitCastInst &I);
void visitAddrSpaceCastInst(AddrSpaceCastInst &I);
@@ -834,6 +836,7 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) {
&GV);
Check(GV.getInitializer()->getType()->isSized(),
"Global variable initializer must be sized", &GV);
+ visitConstantExprsRecursively(GV.getInitializer());
// If the global has common linkage, it must have a zero initializer and
// cannot be constant.
if (GV.hasCommonLinkage()) {
@@ -1296,9 +1299,11 @@ void Verifier::visitDIDerivedType(const DIDerivedType &N) {
if (N.getTag() == dwarf::DW_TAG_set_type) {
if (auto *T = N.getRawBaseType()) {
auto *Enum = dyn_cast_or_null<DICompositeType>(T);
+ auto *Subrange = dyn_cast_or_null<DISubrangeType>(T);
auto *Basic = dyn_cast_or_null<DIBasicType>(T);
CheckDI(
(Enum && Enum->getTag() == dwarf::DW_TAG_enumeration_type) ||
+ (Subrange && Subrange->getTag() == dwarf::DW_TAG_subrange_type) ||
(Basic && (Basic->getEncoding() == dwarf::DW_ATE_unsigned ||
Basic->getEncoding() == dwarf::DW_ATE_signed ||
Basic->getEncoding() == dwarf::DW_ATE_unsigned_char ||
@@ -2610,6 +2615,8 @@ void Verifier::visitConstantExpr(const ConstantExpr *CE) {
Check(CastInst::castIsValid(Instruction::BitCast, CE->getOperand(0),
CE->getType()),
"Invalid bitcast", CE);
+ else if (CE->getOpcode() == Instruction::PtrToAddr)
+ checkPtrToAddr(CE->getOperand(0)->getType(), CE->getType(), *CE);
}
void Verifier::visitConstantPtrAuth(const ConstantPtrAuth *CPA) {
@@ -3002,7 +3009,7 @@ void Verifier::visitFunction(const Function &F) {
if (!IsIntrinsic) {
Check(!Arg.getType()->isMetadataTy(),
"Function takes metadata but isn't an intrinsic", &Arg, &F);
- Check(!Arg.getType()->isTokenTy(),
+ Check(!Arg.getType()->isTokenLikeTy(),
"Function takes token but isn't an intrinsic", &Arg, &F);
Check(!Arg.getType()->isX86_AMXTy(),
"Function takes x86_amx but isn't an intrinsic", &Arg, &F);
@@ -3016,7 +3023,7 @@ void Verifier::visitFunction(const Function &F) {
}
if (!IsIntrinsic) {
- Check(!F.getReturnType()->isTokenTy(),
+ Check(!F.getReturnType()->isTokenLikeTy(),
"Function returns a token but isn't an intrinsic", &F);
Check(!F.getReturnType()->isX86_AMXTy(),
"Function returns a x86_amx but isn't an intrinsic", &F);
@@ -3532,6 +3539,28 @@ void Verifier::visitFPToSIInst(FPToSIInst &I) {
visitInstruction(I);
}
+void Verifier::checkPtrToAddr(Type *SrcTy, Type *DestTy, const Value &V) {
+ Check(SrcTy->isPtrOrPtrVectorTy(), "PtrToAddr source must be pointer", V);
+ Check(DestTy->isIntOrIntVectorTy(), "PtrToAddr result must be integral", V);
+ Check(SrcTy->isVectorTy() == DestTy->isVectorTy(), "PtrToAddr type mismatch",
+ V);
+
+ if (SrcTy->isVectorTy()) {
+ auto *VSrc = cast<VectorType>(SrcTy);
+ auto *VDest = cast<VectorType>(DestTy);
+ Check(VSrc->getElementCount() == VDest->getElementCount(),
+ "PtrToAddr vector length mismatch", V);
+ }
+
+ Type *AddrTy = DL.getAddressType(SrcTy);
+ Check(AddrTy == DestTy, "PtrToAddr result must be address width", V);
+}
+
+void Verifier::visitPtrToAddrInst(PtrToAddrInst &I) {
+ checkPtrToAddr(I.getOperand(0)->getType(), I.getType(), I);
+ visitInstruction(I);
+}
+
void Verifier::visitPtrToIntInst(PtrToIntInst &I) {
// Get the source and destination types
Type *SrcTy = I.getOperand(0)->getType();
@@ -3547,7 +3576,7 @@ void Verifier::visitPtrToIntInst(PtrToIntInst &I) {
auto *VSrc = cast<VectorType>(SrcTy);
auto *VDest = cast<VectorType>(DestTy);
Check(VSrc->getElementCount() == VDest->getElementCount(),
- "PtrToInt Vector width mismatch", &I);
+ "PtrToInt Vector length mismatch", &I);
}
visitInstruction(I);
@@ -3567,7 +3596,7 @@ void Verifier::visitIntToPtrInst(IntToPtrInst &I) {
auto *VSrc = cast<VectorType>(SrcTy);
auto *VDest = cast<VectorType>(DestTy);
Check(VSrc->getElementCount() == VDest->getElementCount(),
- "IntToPtr Vector width mismatch", &I);
+ "IntToPtr Vector length mismatch", &I);
}
visitInstruction(I);
}
@@ -3608,7 +3637,7 @@ void Verifier::visitPHINode(PHINode &PN) {
"PHI nodes not grouped at top of basic block!", &PN, PN.getParent());
// Check that a PHI doesn't yield a Token.
- Check(!PN.getType()->isTokenTy(), "PHI nodes cannot have token type!");
+ Check(!PN.getType()->isTokenLikeTy(), "PHI nodes cannot have token type!");
// Check that all of the values of the PHI node have the same type as the
// result.
@@ -3813,14 +3842,14 @@ void Verifier::visitCallBase(CallBase &Call) {
for (Type *ParamTy : FTy->params()) {
Check(!ParamTy->isMetadataTy(),
"Function has metadata parameter but isn't an intrinsic", Call);
- Check(!ParamTy->isTokenTy(),
+ Check(!ParamTy->isTokenLikeTy(),
"Function has token parameter but isn't an intrinsic", Call);
}
}
// Verify that indirect calls don't return tokens.
if (!Call.getCalledFunction()) {
- Check(!FTy->getReturnType()->isTokenTy(),
+ Check(!FTy->getReturnType()->isTokenLikeTy(),
"Return type cannot be token for indirect call!");
Check(!FTy->getReturnType()->isX86_AMXTy(),
"Return type cannot be x86_amx for indirect call!");
@@ -4609,7 +4638,7 @@ void Verifier::visitEHPadPredecessors(Instruction &I) {
}
// The edge may exit from zero or more nested pads.
- SmallSet<Value *, 8> Seen;
+ SmallPtrSet<Value *, 8> Seen;
for (;; FromPad = getParentPad(FromPad)) {
Check(FromPad != ToPad,
"EH pad cannot handle exceptions raised within it", FromPad, TI);
@@ -4737,7 +4766,7 @@ void Verifier::visitFuncletPadInst(FuncletPadInst &FPI) {
User *FirstUser = nullptr;
Value *FirstUnwindPad = nullptr;
SmallVector<FuncletPadInst *, 8> Worklist({&FPI});
- SmallSet<FuncletPadInst *, 8> Seen;
+ SmallPtrSet<FuncletPadInst *, 8> Seen;
while (!Worklist.empty()) {
FuncletPadInst *CurrentPad = Worklist.pop_back_val();
@@ -4995,6 +5024,13 @@ void Verifier::visitDereferenceableMetadata(Instruction& I, MDNode* MD) {
&I);
}
+void Verifier::visitNofreeMetadata(Instruction &I, MDNode *MD) {
+ Check(I.getType()->isPointerTy(), "nofree applies only to pointer types", &I);
+ Check((isa<IntToPtrInst>(I)), "nofree applies only to inttoptr instruction",
+ &I);
+ Check(MD->getNumOperands() == 0, "nofree metadata must be empty", &I);
+}
+
void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
auto GetBranchingTerminatorNumOperands = [&]() {
unsigned ExpectedNumOperands = 0;
@@ -5470,6 +5506,9 @@ void Verifier::visitInstruction(Instruction &I) {
if (MDNode *MD = I.getMetadata(LLVMContext::MD_dereferenceable_or_null))
visitDereferenceableMetadata(I, MD);
+ if (MDNode *MD = I.getMetadata(LLVMContext::MD_nofree))
+ visitNofreeMetadata(I, MD);
+
if (MDNode *TBAA = I.getMetadata(LLVMContext::MD_tbaa))
TBAAVerifyHelper.visitTBAAMetadata(I, TBAA);
@@ -6612,6 +6651,36 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
"Value for inactive lanes must be a VGPR function argument", &Call);
break;
}
+ case Intrinsic::amdgcn_call_whole_wave: {
+ auto F = dyn_cast<Function>(Call.getArgOperand(0));
+ Check(F, "Indirect whole wave calls are not allowed", &Call);
+
+ CallingConv::ID CC = F->getCallingConv();
+ Check(CC == CallingConv::AMDGPU_Gfx_WholeWave,
+ "Callee must have the amdgpu_gfx_whole_wave calling convention",
+ &Call);
+
+ Check(!F->isVarArg(), "Variadic whole wave calls are not allowed", &Call);
+
+ Check(Call.arg_size() == F->arg_size(),
+ "Call argument count must match callee argument count", &Call);
+
+ // The first argument of the call is the callee, and the first argument of
+ // the callee is the active mask. The rest of the arguments must match.
+ Check(F->arg_begin()->getType()->isIntegerTy(1),
+ "Callee must have i1 as its first argument", &Call);
+ for (auto [CallArg, FuncArg] :
+ drop_begin(zip_equal(Call.args(), F->args()))) {
+ Check(CallArg->getType() == FuncArg.getType(),
+ "Argument types must match", &Call);
+
+ // Check that inreg attributes match between call site and function
+ Check(Call.paramHasAttr(FuncArg.getArgNo(), Attribute::InReg) ==
+ FuncArg.hasInRegAttr(),
+ "Argument inreg attributes must match", &Call);
+ }
+ break;
+ }
case Intrinsic::amdgcn_s_prefetch_data: {
Check(
AMDGPU::isFlatGlobalAddrSpace(
@@ -6668,7 +6737,9 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
"invalid vector type for format", &Call, Src1, Call.getArgOperand(5));
break;
}
- case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4: {
+ case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
+ case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
+ case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
Value *Src0 = Call.getArgOperand(1);
Value *Src1 = Call.getArgOperand(3);
@@ -6716,6 +6787,28 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
"invalid vector type for format", &Call, Src1, Call.getArgOperand(2));
break;
}
+ case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
+ case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
+ case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
+ case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
+ case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
+ case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
+ // Check we only use this intrinsic on the FLAT or GLOBAL address spaces.
+ Value *PtrArg = Call.getArgOperand(0);
+ const unsigned AS = PtrArg->getType()->getPointerAddressSpace();
+ Check(AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS,
+ "cooperative atomic intrinsics require a generic or global pointer",
+ &Call, PtrArg);
+
+ // Last argument must be a MD string
+ auto *Op = cast<MetadataAsValue>(Call.getArgOperand(Call.arg_size() - 1));
+ MDNode *MD = cast<MDNode>(Op->getMetadata());
+ Check((MD->getNumOperands() == 1) && isa<MDString>(MD->getOperand(0)),
+ "cooperative atomic intrinsics require that the last argument is a "
+ "metadata string",
+ &Call, Op);
+ break;
+ }
case Intrinsic::nvvm_setmaxnreg_inc_sync_aligned_u32:
case Intrinsic::nvvm_setmaxnreg_dec_sync_aligned_u32: {
Value *V = Call.getArgOperand(0);
@@ -6769,10 +6862,13 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
break;
}
case Intrinsic::lifetime_start:
- case Intrinsic::lifetime_end:
- Check(isa<AllocaInst>(Call.getArgOperand(1)),
- "llvm.lifetime.start/end can only be used on alloca", &Call);
+ case Intrinsic::lifetime_end: {
+ Value *Ptr = Call.getArgOperand(0);
+ Check(isa<AllocaInst>(Ptr) || isa<PoisonValue>(Ptr),
+ "llvm.lifetime.start/end can only be used on alloca or poison",
+ &Call);
break;
+ }
};
// Verify that there aren't any unmediated control transfers between funclets.
generated by cgit v1.1 at 2025-09-04 18:42:18 +0000