diff options
Diffstat (limited to 'llvm/lib/IR')
36 files changed, 1110 insertions, 228 deletions
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 95d954f..7932765 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -53,6 +53,7 @@ #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" @@ -758,14 +759,12 @@ void TypePrinting::printStructBody(StructType *STy, raw_ostream &OS) { AbstractSlotTrackerStorage::~AbstractSlotTrackerStorage() = default; -namespace llvm { - //===----------------------------------------------------------------------===// // SlotTracker Class: Enumerate slot numbers for unnamed values //===----------------------------------------------------------------------===// /// This class provides computation of slot numbers for LLVM Assembly writing. /// -class SlotTracker : public AbstractSlotTrackerStorage { +class llvm::SlotTracker : public AbstractSlotTrackerStorage { public: /// ValueMap - A mapping of Values to slot numbers. using ValueMap = DenseMap<const Value *, unsigned>; @@ -943,8 +942,6 @@ private: void processDbgRecordMetadata(const DbgRecord &DVR); }; -} // end namespace llvm - ModuleSlotTracker::ModuleSlotTracker(SlotTracker &Machine, const Module *M, const Function *F) : M(M), F(F), Machine(&Machine) {} @@ -1677,12 +1674,14 @@ static void writeConstantInternal(raw_ostream &Out, const Constant *CV, if (const auto *CPA = dyn_cast<ConstantPtrAuth>(CV)) { Out << "ptrauth ("; - // ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC]?]?) + // ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC[, ptr DS]?]?]?) unsigned NumOpsToWrite = 2; if (!CPA->getOperand(2)->isNullValue()) NumOpsToWrite = 3; if (!CPA->getOperand(3)->isNullValue()) NumOpsToWrite = 4; + if (!CPA->getOperand(4)->isNullValue()) + NumOpsToWrite = 5; ListSeparator LS; for (unsigned i = 0, e = NumOpsToWrite; i != e; ++i) { @@ -2935,7 +2934,7 @@ private: // printInfoComment - Print a little comment after the instruction indicating // which slot it occupies. - void printInfoComment(const Value &V); + void printInfoComment(const Value &V, bool isMaterializable = false); // printGCRelocateComment - print comment after call to the gc.relocate // intrinsic indicating base and derived pointer names. @@ -3967,7 +3966,7 @@ void AssemblyWriter::printGlobal(const GlobalVariable *GV) { if (Attrs.hasAttributes()) Out << " #" << Machine.getAttributeGroupSlot(Attrs); - printInfoComment(*GV); + printInfoComment(*GV, GV->isMaterializable()); } void AssemblyWriter::printAlias(const GlobalAlias *GA) { @@ -4005,7 +4004,7 @@ void AssemblyWriter::printAlias(const GlobalAlias *GA) { Out << '"'; } - printInfoComment(*GA); + printInfoComment(*GA, GA->isMaterializable()); Out << '\n'; } @@ -4044,7 +4043,7 @@ void AssemblyWriter::printIFunc(const GlobalIFunc *GI) { printMetadataAttachments(MDs, ", "); } - printInfoComment(*GI); + printInfoComment(*GI, GI->isMaterializable()); Out << '\n'; } @@ -4323,13 +4322,12 @@ void AssemblyWriter::printGCRelocateComment(const GCRelocateInst &Relocate) { /// printInfoComment - Print a little comment after the instruction indicating /// which slot it occupies. -void AssemblyWriter::printInfoComment(const Value &V) { +void AssemblyWriter::printInfoComment(const Value &V, bool isMaterializable) { if (const auto *Relocate = dyn_cast<GCRelocateInst>(&V)) printGCRelocateComment(*Relocate); - if (AnnotationWriter) { + if (AnnotationWriter && !isMaterializable) AnnotationWriter->printInfoComment(V, Out); - } if (PrintInstDebugLocs) { if (auto *I = dyn_cast<Instruction>(&V)) { @@ -4581,12 +4579,38 @@ void AssemblyWriter::printInstruction(const Instruction &I) { Out << ' '; writeOperand(Operand, false); Out << '('; + bool HasPrettyPrintedArgs = + isa<IntrinsicInst>(CI) && + Intrinsic::hasPrettyPrintedArgs(CI->getIntrinsicID()); + ListSeparator LS; - for (unsigned op = 0, Eop = CI->arg_size(); op < Eop; ++op) { - Out << LS; - writeParamOperand(CI->getArgOperand(op), PAL.getParamAttrs(op)); + Function *CalledFunc = CI->getCalledFunction(); + auto PrintArgComment = [&](unsigned ArgNo) { + const auto *ConstArg = dyn_cast<Constant>(CI->getArgOperand(ArgNo)); + if (!ConstArg) + return; + std::string ArgComment; + raw_string_ostream ArgCommentStream(ArgComment); + Intrinsic::ID IID = CalledFunc->getIntrinsicID(); + Intrinsic::printImmArg(IID, ArgNo, ArgCommentStream, ConstArg); + if (ArgComment.empty()) + return; + Out << "/* " << ArgComment << " */ "; + }; + if (HasPrettyPrintedArgs) { + for (unsigned ArgNo = 0, NumArgs = CI->arg_size(); ArgNo < NumArgs; + ++ArgNo) { + Out << LS; + PrintArgComment(ArgNo); + writeParamOperand(CI->getArgOperand(ArgNo), PAL.getParamAttrs(ArgNo)); + } + } else { + for (unsigned ArgNo = 0, NumArgs = CI->arg_size(); ArgNo < NumArgs; + ++ArgNo) { + Out << LS; + writeParamOperand(CI->getArgOperand(ArgNo), PAL.getParamAttrs(ArgNo)); + } } - // Emit an ellipsis if this is a musttail call in a vararg function. This // is only to aid readability, musttail calls forward varargs by default. if (CI->isMustTailCall() && CI->getParent() && @@ -4842,6 +4866,9 @@ void AssemblyWriter::printDbgVariableRecord(const DbgVariableRecord &DVR) { case DbgVariableRecord::LocationType::Declare: Out << "declare"; break; + case DbgVariableRecord::LocationType::DeclareValue: + Out << "declare_value"; + break; case DbgVariableRecord::LocationType::Assign: Out << "assign"; break; @@ -5010,12 +5037,10 @@ void AssemblyWriter::printUseLists(const Function *F) { //===----------------------------------------------------------------------===// void Function::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW, - bool ShouldPreserveUseListOrder, - bool IsForDebug) const { + bool ShouldPreserveUseListOrder, bool IsForDebug) const { SlotTracker SlotTable(this->getParent()); formatted_raw_ostream OS(ROS); - AssemblyWriter W(OS, SlotTable, this->getParent(), AAW, - IsForDebug, + AssemblyWriter W(OS, SlotTable, this->getParent(), AAW, IsForDebug, ShouldPreserveUseListOrder); W.printFunction(this); } diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp index 4ac2ebd..fe6d3e5 100644 --- a/llvm/lib/IR/Attributes.cpp +++ b/llvm/lib/IR/Attributes.cpp @@ -656,6 +656,12 @@ std::string Attribute::getAsString(bool InAttrGrp) const { break; case IRMemLocation::Other: llvm_unreachable("This is represented as the default access kind"); + case IRMemLocation::TargetMem0: + OS << "target_mem0: "; + break; + case IRMemLocation::TargetMem1: + OS << "target_mem1: "; + break; } OS << getModRefStr(MR); } diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index b838e36..2202b08 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -125,6 +125,24 @@ static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, return true; } +// Upgrade the declaration of multipy and add words intrinsics whose input +// arguments' types have changed to vectors of i32 to vectors of i16 +static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, + Function *&NewFn) { + // check if input argument type is a vector of i16 + Type *Arg1Type = F->getFunctionType()->getParamType(1); + Type *Arg2Type = F->getFunctionType()->getParamType(2); + if (Arg1Type->isVectorTy() && + cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) && + Arg2Type->isVectorTy() && + cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16)) + return false; + + rename(F); + NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID); + return true; +} + static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn) { if (F->getReturnType()->getScalarType()->isBFloatTy()) @@ -590,43 +608,89 @@ static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, .Default(Intrinsic::not_intrinsic); if (ID != Intrinsic::not_intrinsic) return upgradeX86MultiplyAddBytes(F, ID, NewFn); + } else if (Name.starts_with("vpdpwssd.") || + Name.starts_with("vpdpwssds.")) { + // Added in 21.1 + ID = StringSwitch<Intrinsic::ID>(Name) + .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128) + .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256) + .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512) + .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128) + .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256) + .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512) + .Default(Intrinsic::not_intrinsic); + if (ID != Intrinsic::not_intrinsic) + return upgradeX86MultiplyAddWords(F, ID, NewFn); } return false; // No other 'x86.avx512.*'. } - if (Name.consume_front("avx2.vpdpb")) { - // Added in 21.1 - ID = StringSwitch<Intrinsic::ID>(Name) - .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128) - .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256) - .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128) - .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256) - .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128) - .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256) - .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128) - .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256) - .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128) - .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256) - .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128) - .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256) - .Default(Intrinsic::not_intrinsic); - if (ID != Intrinsic::not_intrinsic) - return upgradeX86MultiplyAddBytes(F, ID, NewFn); + if (Name.consume_front("avx2.")) { + if (Name.consume_front("vpdpb")) { + // Added in 21.1 + ID = StringSwitch<Intrinsic::ID>(Name) + .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128) + .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256) + .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128) + .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256) + .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128) + .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256) + .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128) + .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256) + .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128) + .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256) + .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128) + .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256) + .Default(Intrinsic::not_intrinsic); + if (ID != Intrinsic::not_intrinsic) + return upgradeX86MultiplyAddBytes(F, ID, NewFn); + } else if (Name.consume_front("vpdpw")) { + // Added in 21.1 + ID = StringSwitch<Intrinsic::ID>(Name) + .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128) + .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256) + .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128) + .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256) + .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128) + .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256) + .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128) + .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256) + .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128) + .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256) + .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128) + .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256) + .Default(Intrinsic::not_intrinsic); + if (ID != Intrinsic::not_intrinsic) + return upgradeX86MultiplyAddWords(F, ID, NewFn); + } return false; // No other 'x86.avx2.*' } - if (Name.consume_front("avx10.vpdpb")) { - // Added in 21.1 - ID = StringSwitch<Intrinsic::ID>(Name) - .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512) - .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512) - .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512) - .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512) - .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512) - .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512) - .Default(Intrinsic::not_intrinsic); - if (ID != Intrinsic::not_intrinsic) - return upgradeX86MultiplyAddBytes(F, ID, NewFn); + if (Name.consume_front("avx10.")) { + if (Name.consume_front("vpdpb")) { + // Added in 21.1 + ID = StringSwitch<Intrinsic::ID>(Name) + .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512) + .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512) + .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512) + .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512) + .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512) + .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512) + .Default(Intrinsic::not_intrinsic); + if (ID != Intrinsic::not_intrinsic) + return upgradeX86MultiplyAddBytes(F, ID, NewFn); + } else if (Name.consume_front("vpdpw")) { + ID = StringSwitch<Intrinsic::ID>(Name) + .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512) + .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512) + .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512) + .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512) + .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512) + .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512) + .Default(Intrinsic::not_intrinsic); + if (ID != Intrinsic::not_intrinsic) + return upgradeX86MultiplyAddWords(F, ID, NewFn); + } return false; // No other 'x86.avx10.*' } @@ -730,7 +794,7 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, // (arm|aarch64).neon.bfdot.*'. Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name) - .Cases("v2f32.v8i8", "v4f32.v16i8", + .Cases({"v2f32.v8i8", "v4f32.v16i8"}, IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot) .Default(Intrinsic::not_intrinsic); @@ -1015,6 +1079,14 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, } return false; // No other 'aarch64.sve.tuple.*'. } + + if (Name.starts_with("rev.nxv")) { + // 'aarch64.sve.rev.<Ty>' + NewFn = Intrinsic::getOrInsertDeclaration( + F->getParent(), Intrinsic::vector_reverse, F->getReturnType()); + return true; + } + return false; // No other 'aarch64.sve.*'. } } @@ -1202,9 +1274,10 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, } if (Name.consume_front("atomic.")) { - if (Name.starts_with("inc") || Name.starts_with("dec")) { - // These were replaced with atomicrmw uinc_wrap and udec_wrap, so - // there's no new declaration. + if (Name.starts_with("inc") || Name.starts_with("dec") || + Name.starts_with("cond.sub") || Name.starts_with("csub")) { + // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond + // and usub_sat so there's no new declaration. NewFn = nullptr; return true; } @@ -1456,7 +1529,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, if (F->arg_size() == 1) { Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name) - .Cases("brev32", "brev64", Intrinsic::bitreverse) + .Cases({"brev32", "brev64"}, Intrinsic::bitreverse) .Case("clz.i", Intrinsic::ctlz) .Case("popc.i", Intrinsic::ctpop) .Default(Intrinsic::not_intrinsic); @@ -1504,6 +1577,10 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, else if (Name.consume_front("fabs.")) // nvvm.fabs.{f,ftz.f,d} Expand = Name == "f" || Name == "ftz.f" || Name == "d"; + else if (Name.consume_front("ex2.approx.")) + // nvvm.ex2.approx.{f,ftz.f,d,f16x2} + Expand = + Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2"; else if (Name.consume_front("max.") || Name.consume_front("min.")) // nvvm.{min,max}.{i,ii,ui,ull} Expand = Name == "s" || Name == "i" || Name == "ll" || Name == "us" || @@ -2550,6 +2627,11 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz : Intrinsic::nvvm_fabs; Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0)); + } else if (Name.consume_front("ex2.approx.")) { + // nvvm.ex2.approx.{f,ftz.f,d,f16x2} + Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz + : Intrinsic::nvvm_ex2_approx; + Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0)); } else if (Name.starts_with("atomic.load.add.f32.p") || Name.starts_with("atomic.load.add.f64.p")) { Value *Ptr = CI->getArgOperand(0); @@ -4298,6 +4380,32 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2)}; + + // Input arguments types were incorrectly set to vectors of i32 before but + // they should be vectors of i16. Insert bit cast when encountering the old + // types + if (Args[1]->getType()->isVectorTy() && + cast<VectorType>(Args[1]->getType()) + ->getElementType() + ->isIntegerTy(32) && + Args[2]->getType()->isVectorTy() && + cast<VectorType>(Args[2]->getType()) + ->getElementType() + ->isIntegerTy(32)) { + Type *NewArgType = nullptr; + if (VecWidth == 128) + NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false); + else if (VecWidth == 256) + NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false); + else if (VecWidth == 512) + NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false); + else + llvm_unreachable("Unexpected vector bit width"); + + Args[1] = Builder.CreateBitCast(Args[1], NewArgType); + Args[2] = Builder.CreateBitCast(Args[2], NewArgType); + } + Rep = Builder.CreateIntrinsic(IID, Args); Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) : CI->getArgOperand(0); @@ -4499,7 +4607,9 @@ static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin) .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin) .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax) - .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax); + .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax) + .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond) + .StartsWith("atomic.csub", AtomicRMWInst::USubSat); unsigned NumOperands = CI->getNumOperands(); if (NumOperands < 3) // Malformed bitcode. @@ -5373,6 +5483,39 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { NewCall = Builder.CreateCall(NewFn, Args); break; } + case Intrinsic::x86_avx512_vpdpwssd_128: + case Intrinsic::x86_avx512_vpdpwssd_256: + case Intrinsic::x86_avx512_vpdpwssd_512: + case Intrinsic::x86_avx512_vpdpwssds_128: + case Intrinsic::x86_avx512_vpdpwssds_256: + case Intrinsic::x86_avx512_vpdpwssds_512: + case Intrinsic::x86_avx2_vpdpwsud_128: + case Intrinsic::x86_avx2_vpdpwsud_256: + case Intrinsic::x86_avx10_vpdpwsud_512: + case Intrinsic::x86_avx2_vpdpwsuds_128: + case Intrinsic::x86_avx2_vpdpwsuds_256: + case Intrinsic::x86_avx10_vpdpwsuds_512: + case Intrinsic::x86_avx2_vpdpwusd_128: + case Intrinsic::x86_avx2_vpdpwusd_256: + case Intrinsic::x86_avx10_vpdpwusd_512: + case Intrinsic::x86_avx2_vpdpwusds_128: + case Intrinsic::x86_avx2_vpdpwusds_256: + case Intrinsic::x86_avx10_vpdpwusds_512: + case Intrinsic::x86_avx2_vpdpwuud_128: + case Intrinsic::x86_avx2_vpdpwuud_256: + case Intrinsic::x86_avx10_vpdpwuud_512: + case Intrinsic::x86_avx2_vpdpwuuds_128: + case Intrinsic::x86_avx2_vpdpwuuds_256: + case Intrinsic::x86_avx10_vpdpwuuds_512: + unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16; + Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), + CI->getArgOperand(2)}; + Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false); + Args[1] = Builder.CreateBitCast(Args[1], NewArgType); + Args[2] = Builder.CreateBitCast(Args[2], NewArgType); + + NewCall = Builder.CreateCall(NewFn, Args); + break; } assert(NewCall && "Should have either set this variable or returned through " "the default case"); diff --git a/llvm/lib/IR/CMakeLists.txt b/llvm/lib/IR/CMakeLists.txt index 10572ff..31821a2 100644 --- a/llvm/lib/IR/CMakeLists.txt +++ b/llvm/lib/IR/CMakeLists.txt @@ -35,6 +35,7 @@ add_llvm_component_library(LLVMCore GVMaterializer.cpp Globals.cpp Intrinsics.cpp + NVVMIntrinsicUtils.cpp IRBuilder.cpp IRPrintingPasses.cpp SSAContext.cpp diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp index b454c9a..9beaee6 100644 --- a/llvm/lib/IR/ConstantRange.cpp +++ b/llvm/lib/IR/ConstantRange.cpp @@ -841,6 +841,8 @@ ConstantRange ConstantRange::zeroExtend(uint32_t DstTySize) const { if (isEmptySet()) return getEmpty(DstTySize); unsigned SrcTySize = getBitWidth(); + if (DstTySize == SrcTySize) + return *this; assert(SrcTySize < DstTySize && "Not a value extension"); if (isFullSet() || isUpperWrapped()) { // Change into [0, 1 << src bit width) @@ -858,6 +860,8 @@ ConstantRange ConstantRange::signExtend(uint32_t DstTySize) const { if (isEmptySet()) return getEmpty(DstTySize); unsigned SrcTySize = getBitWidth(); + if (DstTySize == SrcTySize) + return *this; assert(SrcTySize < DstTySize && "Not a value extension"); // special case: [X, INT_MIN) -- not really wrapping around @@ -874,6 +878,8 @@ ConstantRange ConstantRange::signExtend(uint32_t DstTySize) const { ConstantRange ConstantRange::truncate(uint32_t DstTySize, unsigned NoWrapKind) const { + if (DstTySize == getBitWidth()) + return *this; assert(getBitWidth() > DstTySize && "Not a value truncation"); if (isEmptySet()) return getEmpty(DstTySize); diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp index cbce8bd..2f02027 100644 --- a/llvm/lib/IR/Constants.cpp +++ b/llvm/lib/IR/Constants.cpp @@ -183,6 +183,23 @@ bool Constant::isMinSignedValue() const { return false; } +bool Constant::isMaxSignedValue() const { + // Check for INT_MAX integers + if (const ConstantInt *CI = dyn_cast<ConstantInt>(this)) + return CI->isMaxValue(/*isSigned=*/true); + + // Check for FP which are bitcasted from INT_MAX integers + if (const ConstantFP *CFP = dyn_cast<ConstantFP>(this)) + return CFP->getValueAPF().bitcastToAPInt().isMaxSignedValue(); + + // Check for splats of INT_MAX values. + if (getType()->isVectorTy()) + if (const auto *SplatVal = getSplatValue()) + return SplatVal->isMaxSignedValue(); + + return false; +} + bool Constant::isNotMinSignedValue() const { // Check for INT_MIN integers if (const ConstantInt *CI = dyn_cast<ConstantInt>(this)) @@ -943,8 +960,10 @@ ConstantInt *ConstantInt::get(LLVMContext &Context, ElementCount EC, return Slot.get(); } -Constant *ConstantInt::get(Type *Ty, uint64_t V, bool isSigned) { - Constant *C = get(cast<IntegerType>(Ty->getScalarType()), V, isSigned); +Constant *ConstantInt::get(Type *Ty, uint64_t V, bool IsSigned, + bool ImplicitTrunc) { + Constant *C = + get(cast<IntegerType>(Ty->getScalarType()), V, IsSigned, ImplicitTrunc); // For vectors, broadcast the value. if (VectorType *VTy = dyn_cast<VectorType>(Ty)) @@ -953,11 +972,10 @@ Constant *ConstantInt::get(Type *Ty, uint64_t V, bool isSigned) { return C; } -ConstantInt *ConstantInt::get(IntegerType *Ty, uint64_t V, bool isSigned) { - // TODO: Avoid implicit trunc? - // See https://github.com/llvm/llvm-project/issues/112510. +ConstantInt *ConstantInt::get(IntegerType *Ty, uint64_t V, bool IsSigned, + bool ImplicitTrunc) { return get(Ty->getContext(), - APInt(Ty->getBitWidth(), V, isSigned, /*implicitTrunc=*/true)); + APInt(Ty->getBitWidth(), V, IsSigned, ImplicitTrunc)); } Constant *ConstantInt::get(Type *Ty, const APInt& V) { @@ -2064,28 +2082,33 @@ Value *NoCFIValue::handleOperandChangeImpl(Value *From, Value *To) { // ConstantPtrAuth *ConstantPtrAuth::get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) { - Constant *ArgVec[] = {Ptr, Key, Disc, AddrDisc}; + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) { + Constant *ArgVec[] = {Ptr, Key, Disc, AddrDisc, DeactivationSymbol}; ConstantPtrAuthKeyType MapKey(ArgVec); LLVMContextImpl *pImpl = Ptr->getContext().pImpl; return pImpl->ConstantPtrAuths.getOrCreate(Ptr->getType(), MapKey); } ConstantPtrAuth *ConstantPtrAuth::getWithSameSchema(Constant *Pointer) const { - return get(Pointer, getKey(), getDiscriminator(), getAddrDiscriminator()); + return get(Pointer, getKey(), getDiscriminator(), getAddrDiscriminator(), + getDeactivationSymbol()); } ConstantPtrAuth::ConstantPtrAuth(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) : Constant(Ptr->getType(), Value::ConstantPtrAuthVal, AllocMarker) { assert(Ptr->getType()->isPointerTy()); assert(Key->getBitWidth() == 32); assert(Disc->getBitWidth() == 64); assert(AddrDisc->getType()->isPointerTy()); + assert(DeactivationSymbol->getType()->isPointerTy()); setOperand(0, Ptr); setOperand(1, Key); setOperand(2, Disc); setOperand(3, AddrDisc); + setOperand(4, DeactivationSymbol); } /// Remove the constant from the constant table. @@ -2133,6 +2156,11 @@ bool ConstantPtrAuth::hasSpecialAddressDiscriminator(uint64_t Value) const { bool ConstantPtrAuth::isKnownCompatibleWith(const Value *Key, const Value *Discriminator, const DataLayout &DL) const { + // This function may only be validly called to analyze a ptrauth operation + // with no deactivation symbol, so if we have one it isn't compatible. + if (!getDeactivationSymbol()->isNullValue()) + return false; + // If the keys are different, there's no chance for this to be compatible. if (getKey() != Key) return false; diff --git a/llvm/lib/IR/ConstantsContext.h b/llvm/lib/IR/ConstantsContext.h index 51fb40b..2073e0d 100644 --- a/llvm/lib/IR/ConstantsContext.h +++ b/llvm/lib/IR/ConstantsContext.h @@ -535,11 +535,12 @@ struct ConstantPtrAuthKeyType { unsigned getHash() const { return hash_combine_range(Operands); } - using TypeClass = typename ConstantInfo<ConstantPtrAuth>::TypeClass; + using TypeClass = ConstantInfo<ConstantPtrAuth>::TypeClass; ConstantPtrAuth *create(TypeClass *Ty) const { return new ConstantPtrAuth(Operands[0], cast<ConstantInt>(Operands[1]), - cast<ConstantInt>(Operands[2]), Operands[3]); + cast<ConstantInt>(Operands[2]), Operands[3], + Operands[4]); } }; diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp index 27d8294..851aeb0 100644 --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -97,11 +97,15 @@ static LLVMContext &getGlobalContext() { return GlobalContext; } +LLVMContextRef llvm::getGlobalContextForCAPI() { + return wrap(&getGlobalContext()); +} + LLVMContextRef LLVMContextCreate() { return wrap(new LLVMContext()); } -LLVMContextRef LLVMGetGlobalContext() { return wrap(&getGlobalContext()); } +LLVMContextRef LLVMGetGlobalContext() { return getGlobalContextForCAPI(); } void LLVMContextSetDiagnosticHandler(LLVMContextRef C, LLVMDiagnosticHandler Handler, @@ -146,7 +150,7 @@ unsigned LLVMGetMDKindIDInContext(LLVMContextRef C, const char *Name, } unsigned LLVMGetMDKindID(const char *Name, unsigned SLen) { - return LLVMGetMDKindIDInContext(LLVMGetGlobalContext(), Name, SLen); + return LLVMGetMDKindIDInContext(getGlobalContextForCAPI(), Name, SLen); } unsigned LLVMGetSyncScopeID(LLVMContextRef C, const char *Name, size_t SLen) { @@ -681,25 +685,25 @@ LLVMTypeRef LLVMIntTypeInContext(LLVMContextRef C, unsigned NumBits) { } LLVMTypeRef LLVMInt1Type(void) { - return LLVMInt1TypeInContext(LLVMGetGlobalContext()); + return LLVMInt1TypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMInt8Type(void) { - return LLVMInt8TypeInContext(LLVMGetGlobalContext()); + return LLVMInt8TypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMInt16Type(void) { - return LLVMInt16TypeInContext(LLVMGetGlobalContext()); + return LLVMInt16TypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMInt32Type(void) { - return LLVMInt32TypeInContext(LLVMGetGlobalContext()); + return LLVMInt32TypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMInt64Type(void) { - return LLVMInt64TypeInContext(LLVMGetGlobalContext()); + return LLVMInt64TypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMInt128Type(void) { - return LLVMInt128TypeInContext(LLVMGetGlobalContext()); + return LLVMInt128TypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMIntType(unsigned NumBits) { - return LLVMIntTypeInContext(LLVMGetGlobalContext(), NumBits); + return LLVMIntTypeInContext(getGlobalContextForCAPI(), NumBits); } unsigned LLVMGetIntTypeWidth(LLVMTypeRef IntegerTy) { @@ -734,28 +738,28 @@ LLVMTypeRef LLVMX86AMXTypeInContext(LLVMContextRef C) { } LLVMTypeRef LLVMHalfType(void) { - return LLVMHalfTypeInContext(LLVMGetGlobalContext()); + return LLVMHalfTypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMBFloatType(void) { - return LLVMBFloatTypeInContext(LLVMGetGlobalContext()); + return LLVMBFloatTypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMFloatType(void) { - return LLVMFloatTypeInContext(LLVMGetGlobalContext()); + return LLVMFloatTypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMDoubleType(void) { - return LLVMDoubleTypeInContext(LLVMGetGlobalContext()); + return LLVMDoubleTypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMX86FP80Type(void) { - return LLVMX86FP80TypeInContext(LLVMGetGlobalContext()); + return LLVMX86FP80TypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMFP128Type(void) { - return LLVMFP128TypeInContext(LLVMGetGlobalContext()); + return LLVMFP128TypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMPPCFP128Type(void) { - return LLVMPPCFP128TypeInContext(LLVMGetGlobalContext()); + return LLVMPPCFP128TypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMX86AMXType(void) { - return LLVMX86AMXTypeInContext(LLVMGetGlobalContext()); + return LLVMX86AMXTypeInContext(getGlobalContextForCAPI()); } /*--.. Operations on function types ........................................--*/ @@ -795,7 +799,7 @@ LLVMTypeRef LLVMStructTypeInContext(LLVMContextRef C, LLVMTypeRef *ElementTypes, LLVMTypeRef LLVMStructType(LLVMTypeRef *ElementTypes, unsigned ElementCount, LLVMBool Packed) { - return LLVMStructTypeInContext(LLVMGetGlobalContext(), ElementTypes, + return LLVMStructTypeInContext(getGlobalContextForCAPI(), ElementTypes, ElementCount, Packed); } @@ -952,10 +956,10 @@ LLVMTypeRef LLVMMetadataTypeInContext(LLVMContextRef C) { } LLVMTypeRef LLVMVoidType(void) { - return LLVMVoidTypeInContext(LLVMGetGlobalContext()); + return LLVMVoidTypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMLabelType(void) { - return LLVMLabelTypeInContext(LLVMGetGlobalContext()); + return LLVMLabelTypeInContext(getGlobalContextForCAPI()); } LLVMTypeRef LLVMTargetExtTypeInContext(LLVMContextRef C, const char *Name, @@ -1296,7 +1300,7 @@ LLVMValueRef LLVMMDStringInContext(LLVMContextRef C, const char *Str, } LLVMValueRef LLVMMDString(const char *Str, unsigned SLen) { - return LLVMMDStringInContext(LLVMGetGlobalContext(), Str, SLen); + return LLVMMDStringInContext(getGlobalContextForCAPI(), Str, SLen); } LLVMValueRef LLVMMDNodeInContext(LLVMContextRef C, LLVMValueRef *Vals, @@ -1327,7 +1331,7 @@ LLVMValueRef LLVMMDNodeInContext(LLVMContextRef C, LLVMValueRef *Vals, } LLVMValueRef LLVMMDNode(LLVMValueRef *Vals, unsigned Count) { - return LLVMMDNodeInContext(LLVMGetGlobalContext(), Vals, Count); + return LLVMMDNodeInContext(getGlobalContextForCAPI(), Vals, Count); } LLVMValueRef LLVMMetadataAsValue(LLVMContextRef C, LLVMMetadataRef MD) { @@ -1573,6 +1577,14 @@ LLVMValueRef LLVMConstRealOfStringAndSize(LLVMTypeRef RealTy, const char Str[], return wrap(ConstantFP::get(unwrap(RealTy), StringRef(Str, SLen))); } +LLVMValueRef LLVMConstFPFromBits(LLVMTypeRef Ty, const uint64_t N[]) { + Type *T = unwrap(Ty); + unsigned SB = T->getScalarSizeInBits(); + APInt AI(SB, ArrayRef<uint64_t>(N, divideCeil(SB, 64))); + APFloat Quad(T->getFltSemantics(), AI); + return wrap(ConstantFP::get(T, Quad)); +} + unsigned long long LLVMConstIntGetZExtValue(LLVMValueRef ConstantVal) { return unwrap<ConstantInt>(ConstantVal)->getZExtValue(); } @@ -1620,7 +1632,7 @@ LLVMValueRef LLVMConstStringInContext2(LLVMContextRef C, const char *Str, LLVMValueRef LLVMConstString(const char *Str, unsigned Length, LLVMBool DontNullTerminate) { - return LLVMConstStringInContext(LLVMGetGlobalContext(), Str, Length, + return LLVMConstStringInContext(getGlobalContextForCAPI(), Str, Length, DontNullTerminate); } @@ -1677,8 +1689,8 @@ LLVMValueRef LLVMConstStructInContext(LLVMContextRef C, LLVMValueRef LLVMConstStruct(LLVMValueRef *ConstantVals, unsigned Count, LLVMBool Packed) { - return LLVMConstStructInContext(LLVMGetGlobalContext(), ConstantVals, Count, - Packed); + return LLVMConstStructInContext(getGlobalContextForCAPI(), ConstantVals, + Count, Packed); } LLVMValueRef LLVMConstNamedStruct(LLVMTypeRef StructTy, @@ -1699,7 +1711,9 @@ LLVMValueRef LLVMConstantPtrAuth(LLVMValueRef Ptr, LLVMValueRef Key, LLVMValueRef Disc, LLVMValueRef AddrDisc) { return wrap(ConstantPtrAuth::get( unwrap<Constant>(Ptr), unwrap<ConstantInt>(Key), - unwrap<ConstantInt>(Disc), unwrap<Constant>(AddrDisc))); + unwrap<ConstantInt>(Disc), unwrap<Constant>(AddrDisc), + ConstantPointerNull::get( + cast<PointerType>(unwrap<Constant>(AddrDisc)->getType())))); } /*-- Opcode mapping */ @@ -2882,7 +2896,7 @@ LLVMBasicBlockRef LLVMAppendBasicBlockInContext(LLVMContextRef C, } LLVMBasicBlockRef LLVMAppendBasicBlock(LLVMValueRef FnRef, const char *Name) { - return LLVMAppendBasicBlockInContext(LLVMGetGlobalContext(), FnRef, Name); + return LLVMAppendBasicBlockInContext(getGlobalContextForCAPI(), FnRef, Name); } LLVMBasicBlockRef LLVMInsertBasicBlockInContext(LLVMContextRef C, @@ -2894,7 +2908,7 @@ LLVMBasicBlockRef LLVMInsertBasicBlockInContext(LLVMContextRef C, LLVMBasicBlockRef LLVMInsertBasicBlock(LLVMBasicBlockRef BBRef, const char *Name) { - return LLVMInsertBasicBlockInContext(LLVMGetGlobalContext(), BBRef, Name); + return LLVMInsertBasicBlockInContext(getGlobalContextForCAPI(), BBRef, Name); } void LLVMDeleteBasicBlock(LLVMBasicBlockRef BBRef) { @@ -3036,6 +3050,37 @@ LLVMDbgRecordRef LLVMGetPreviousDbgRecord(LLVMDbgRecordRef Rec) { return wrap(&*--I); } +LLVMMetadataRef LLVMDbgRecordGetDebugLoc(LLVMDbgRecordRef Rec) { + return wrap(unwrap<DbgRecord>(Rec)->getDebugLoc().getAsMDNode()); +} + +LLVMDbgRecordKind LLVMDbgRecordGetKind(LLVMDbgRecordRef Rec) { + DbgRecord *Record = unwrap<DbgRecord>(Rec); + if (isa<DbgLabelRecord>(Record)) + return LLVMDbgRecordLabel; + DbgVariableRecord *VariableRecord = dyn_cast<DbgVariableRecord>(Record); + assert(VariableRecord && "unexpected record"); + if (VariableRecord->isDbgDeclare()) + return LLVMDbgRecordDeclare; + if (VariableRecord->isDbgValue()) + return LLVMDbgRecordValue; + assert(VariableRecord->isDbgAssign() && "unexpected record"); + return LLVMDbgRecordAssign; +} + +LLVMValueRef LLVMDbgVariableRecordGetValue(LLVMDbgRecordRef Rec, + unsigned OpIdx) { + return wrap(unwrap<DbgVariableRecord>(Rec)->getValue(OpIdx)); +} + +LLVMMetadataRef LLVMDbgVariableRecordGetVariable(LLVMDbgRecordRef Rec) { + return wrap(unwrap<DbgVariableRecord>(Rec)->getRawVariable()); +} + +LLVMMetadataRef LLVMDbgVariableRecordGetExpression(LLVMDbgRecordRef Rec) { + return wrap(unwrap<DbgVariableRecord>(Rec)->getRawExpression()); +} + unsigned LLVMGetNumArgOperands(LLVMValueRef Instr) { if (FuncletPadInst *FPI = dyn_cast<FuncletPadInst>(unwrap(Instr))) { return FPI->arg_size(); @@ -3299,7 +3344,7 @@ LLVMBuilderRef LLVMCreateBuilderInContext(LLVMContextRef C) { } LLVMBuilderRef LLVMCreateBuilder(void) { - return LLVMCreateBuilderInContext(LLVMGetGlobalContext()); + return LLVMCreateBuilderInContext(getGlobalContextForCAPI()); } static void LLVMPositionBuilderImpl(IRBuilder<> *Builder, BasicBlock *Block, diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp index ca11ecf..b01860d 100644 --- a/llvm/lib/IR/DIBuilder.cpp +++ b/llvm/lib/IR/DIBuilder.cpp @@ -1147,6 +1147,24 @@ DbgInstPtr DIBuilder::insertDeclare(Value *Storage, DILocalVariable *VarInfo, return DVR; } +DbgInstPtr DIBuilder::insertDeclareValue(Value *Storage, + DILocalVariable *VarInfo, + DIExpression *Expr, + const DILocation *DL, + InsertPosition InsertPt) { + assert(VarInfo && + "empty or invalid DILocalVariable* passed to dbg.declare_value"); + assert(DL && "Expected debug loc"); + assert(DL->getScope()->getSubprogram() == + VarInfo->getScope()->getSubprogram() && + "Expected matching subprograms"); + + DbgVariableRecord *DVR = + DbgVariableRecord::createDVRDeclareValue(Storage, VarInfo, Expr, DL); + insertDbgVariableRecord(DVR, InsertPt); + return DVR; +} + void DIBuilder::insertDbgVariableRecord(DbgVariableRecord *DVR, InsertPosition InsertPt) { assert(InsertPt.isValid()); diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index 5883606..1859bc4 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -40,7 +40,6 @@ #include <algorithm> #include <cassert> #include <optional> -#include <utility> using namespace llvm; using namespace llvm::at; @@ -63,6 +62,23 @@ TinyPtrVector<DbgVariableRecord *> llvm::findDVRDeclares(Value *V) { return Declares; } +TinyPtrVector<DbgVariableRecord *> llvm::findDVRDeclareValues(Value *V) { + // This function is hot. Check whether the value has any metadata to avoid a + // DenseMap lookup. This check is a bitfield datamember lookup. + if (!V->isUsedByMetadata()) + return {}; + auto *L = ValueAsMetadata::getIfExists(V); + if (!L) + return {}; + + TinyPtrVector<DbgVariableRecord *> DEclareValues; + for (DbgVariableRecord *DVR : L->getAllDbgVariableRecordUsers()) + if (DVR->getType() == DbgVariableRecord::LocationType::DeclareValue) + DEclareValues.push_back(DVR); + + return DEclareValues; +} + TinyPtrVector<DbgVariableRecord *> llvm::findDVRValues(Value *V) { // This function is hot. Check whether the value has any metadata to avoid a // DenseMap lookup. This check is a bitfield datamember lookup. @@ -247,7 +263,7 @@ void DebugInfoFinder::processType(DIType *DT) { } } -void DebugInfoFinder::processImportedEntity(DIImportedEntity *Import) { +void DebugInfoFinder::processImportedEntity(const DIImportedEntity *Import) { auto *Entity = Import->getEntity(); if (auto *T = dyn_cast<DIType>(Entity)) processType(T); @@ -307,15 +323,13 @@ void DebugInfoFinder::processSubprogram(DISubprogram *SP) { } } - for (auto *N : SP->getRetainedNodes()) { - if (auto *Var = dyn_cast_or_null<DILocalVariable>(N)) - processVariable(Var); - else if (auto *Import = dyn_cast_or_null<DIImportedEntity>(N)) - processImportedEntity(Import); - } + SP->forEachRetainedNode( + [this](const DILocalVariable *LV) { processVariable(LV); }, + [](const DILabel *L) {}, + [this](const DIImportedEntity *IE) { processImportedEntity(IE); }); } -void DebugInfoFinder::processVariable(DILocalVariable *DV) { +void DebugInfoFinder::processVariable(const DILocalVariable *DV) { if (!NodesSeen.insert(DV).second) return; processScope(DV->getScope()); diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp index fafc325..79b512d 100644 --- a/llvm/lib/IR/DebugInfoMetadata.cpp +++ b/llvm/lib/IR/DebugInfoMetadata.cpp @@ -839,7 +839,7 @@ DISubrangeType::convertRawToBound(Metadata *IN) const { return BoundType(); assert(isa<ConstantAsMetadata>(IN) || isa<DIVariable>(IN) || - isa<DIExpression>(IN)); + isa<DIExpression>(IN) || isa<DIDerivedType>(IN)); if (auto *MD = dyn_cast<ConstantAsMetadata>(IN)) return BoundType(cast<ConstantInt>(MD->getValue())); @@ -850,6 +850,9 @@ DISubrangeType::convertRawToBound(Metadata *IN) const { if (auto *MD = dyn_cast<DIExpression>(IN)) return BoundType(MD); + if (auto *DT = dyn_cast<DIDerivedType>(IN)) + return BoundType(DT); + return BoundType(); } @@ -962,16 +965,29 @@ DIType *DIDerivedType::getClassType() const { assert(getTag() == dwarf::DW_TAG_ptr_to_member_type); return cast_or_null<DIType>(getExtraData()); } + +// Helper function to extract ConstantAsMetadata from ExtraData, +// handling extra data MDTuple unwrapping if needed. +static ConstantAsMetadata *extractConstantMetadata(Metadata *ExtraData) { + Metadata *ED = ExtraData; + if (auto *Tuple = dyn_cast_or_null<MDTuple>(ED)) { + if (Tuple->getNumOperands() != 1) + return nullptr; + ED = Tuple->getOperand(0); + } + return cast_or_null<ConstantAsMetadata>(ED); +} + uint32_t DIDerivedType::getVBPtrOffset() const { assert(getTag() == dwarf::DW_TAG_inheritance); - if (auto *CM = cast_or_null<ConstantAsMetadata>(getExtraData())) + if (auto *CM = extractConstantMetadata(getExtraData())) if (auto *CI = dyn_cast_or_null<ConstantInt>(CM->getValue())) return static_cast<uint32_t>(CI->getZExtValue()); return 0; } Constant *DIDerivedType::getStorageOffsetInBits() const { assert(getTag() == dwarf::DW_TAG_member && isBitField()); - if (auto *C = cast_or_null<ConstantAsMetadata>(getExtraData())) + if (auto *C = extractConstantMetadata(getExtraData())) return C->getValue(); return nullptr; } @@ -980,13 +996,13 @@ Constant *DIDerivedType::getConstant() const { assert((getTag() == dwarf::DW_TAG_member || getTag() == dwarf::DW_TAG_variable) && isStaticMember()); - if (auto *C = cast_or_null<ConstantAsMetadata>(getExtraData())) + if (auto *C = extractConstantMetadata(getExtraData())) return C->getValue(); return nullptr; } Constant *DIDerivedType::getDiscriminantValue() const { assert(getTag() == dwarf::DW_TAG_member && !isStaticMember()); - if (auto *C = cast_or_null<ConstantAsMetadata>(getExtraData())) + if (auto *C = extractConstantMetadata(getExtraData())) return C->getValue(); return nullptr; } @@ -1428,6 +1444,19 @@ bool DISubprogram::describes(const Function *F) const { assert(F && "Invalid function"); return F->getSubprogram() == this; } + +const DIScope *DISubprogram::getRawRetainedNodeScope(const MDNode *N) { + return visitRetainedNode<DIScope *>( + N, [](const DILocalVariable *LV) { return LV->getScope(); }, + [](const DILabel *L) { return L->getScope(); }, + [](const DIImportedEntity *IE) { return IE->getScope(); }, + [](const Metadata *N) { return nullptr; }); +} + +const DILocalScope *DISubprogram::getRetainedNodeScope(const MDNode *N) { + return cast<DILocalScope>(getRawRetainedNodeScope(N)); +} + DILexicalBlockBase::DILexicalBlockBase(LLVMContext &C, unsigned ID, StorageType Storage, ArrayRef<Metadata *> Ops) diff --git a/llvm/lib/IR/DebugLoc.cpp b/llvm/lib/IR/DebugLoc.cpp index 01dafca..a3c98ae 100644 --- a/llvm/lib/IR/DebugLoc.cpp +++ b/llvm/lib/IR/DebugLoc.cpp @@ -10,10 +10,11 @@ #include "llvm/Config/llvm-config.h" #include "llvm/IR/DebugInfo.h" +using namespace llvm; + #if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN #include "llvm/Support/Signals.h" -namespace llvm { DbgLocOrigin::DbgLocOrigin(bool ShouldCollectTrace) { if (!ShouldCollectTrace) return; @@ -30,10 +31,7 @@ void DbgLocOrigin::addTrace() { auto &[Depth, StackTrace] = StackTraces.emplace_back(); Depth = sys::getStackTrace(StackTrace); } -} // namespace llvm -#endif - -using namespace llvm; +#endif // LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN #if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE DILocAndCoverageTracking::DILocAndCoverageTracking(const DILocation *L) @@ -85,16 +83,14 @@ DebugLoc DebugLoc::getFnDebugLoc() const { } bool DebugLoc::isImplicitCode() const { - if (DILocation *Loc = get()) { + if (DILocation *Loc = get()) return Loc->isImplicitCode(); - } return true; } void DebugLoc::setImplicitCode(bool ImplicitCode) { - if (DILocation *Loc = get()) { + if (DILocation *Loc = get()) Loc->setImplicitCode(ImplicitCode); - } } DebugLoc DebugLoc::replaceInlinedAtSubprogram( diff --git a/llvm/lib/IR/DebugProgramInstruction.cpp b/llvm/lib/IR/DebugProgramInstruction.cpp index d9357bb..9efa3c7 100644 --- a/llvm/lib/IR/DebugProgramInstruction.cpp +++ b/llvm/lib/IR/DebugProgramInstruction.cpp @@ -12,8 +12,9 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/Support/Compiler.h" -namespace llvm { +using namespace llvm; +namespace llvm { template <typename T> DbgRecordParamRef<T>::DbgRecordParamRef(const T *Param) : Ref(const_cast<T *>(Param)) {} @@ -28,6 +29,7 @@ template <typename T> T *DbgRecordParamRef<T>::get() const { template class LLVM_EXPORT_TEMPLATE DbgRecordParamRef<DIExpression>; template class LLVM_EXPORT_TEMPLATE DbgRecordParamRef<DILabel>; template class LLVM_EXPORT_TEMPLATE DbgRecordParamRef<DILocalVariable>; +} // namespace llvm DbgVariableRecord::DbgVariableRecord(const DbgVariableIntrinsic *DVI) : DbgRecord(ValueKind, DVI->getDebugLoc()), @@ -209,6 +211,22 @@ DbgVariableRecord::createDVRDeclare(Value *Address, DILocalVariable *DV, return NewDVRDeclare; } +DbgVariableRecord * +DbgVariableRecord::createDVRDeclareValue(Value *Address, DILocalVariable *DV, + DIExpression *Expr, + const DILocation *DI) { + return new DbgVariableRecord(ValueAsMetadata::get(Address), DV, Expr, DI, + LocationType::DeclareValue); +} + +DbgVariableRecord *DbgVariableRecord::createDVRDeclareValue( + Value *Address, DILocalVariable *DV, DIExpression *Expr, + const DILocation *DI, DbgVariableRecord &InsertBefore) { + auto *NewDVRCoro = createDVRDeclareValue(Address, DV, Expr, DI); + NewDVRCoro->insertBefore(&InsertBefore); + return NewDVRCoro; +} + DbgVariableRecord *DbgVariableRecord::createDVRAssign( Value *Val, DILocalVariable *Variable, DIExpression *Expression, DIAssignID *AssignID, Value *Address, DIExpression *AddressExpression, @@ -425,6 +443,10 @@ DbgVariableRecord::createDebugIntrinsic(Module *M, case DbgVariableRecord::LocationType::End: case DbgVariableRecord::LocationType::Any: llvm_unreachable("Invalid LocationType"); + break; + case DbgVariableRecord::LocationType::DeclareValue: + llvm_unreachable( + "#dbg_declare_value should never be converted to an intrinsic"); } // Create the intrinsic from this DbgVariableRecord's information, optionally @@ -755,5 +777,3 @@ iterator_range<simple_ilist<DbgRecord>::iterator> DbgMarker::cloneDebugInfoFrom( // We inserted a block at the end, return that range. return {First->getIterator(), StoredDbgRecords.end()}; } - -} // end namespace llvm diff --git a/llvm/lib/IR/EHPersonalities.cpp b/llvm/lib/IR/EHPersonalities.cpp index 9297a82..12ae474 100644 --- a/llvm/lib/IR/EHPersonalities.cpp +++ b/llvm/lib/IR/EHPersonalities.cpp @@ -47,7 +47,8 @@ EHPersonality llvm::classifyEHPersonality(const Value *Pers) { .Case("__C_specific_handler", EHPersonality::MSVC_TableSEH) .Case("__CxxFrameHandler3", EHPersonality::MSVC_CXX) .Case("ProcessCLRException", EHPersonality::CoreCLR) - .Case("rust_eh_personality", EHPersonality::Rust) + // Rust mangles its personality function, so we can't test exact equality. + .EndsWith("rust_eh_personality", EHPersonality::Rust) .Case("__gxx_wasm_personality_v0", EHPersonality::Wasm_CXX) .Case("__xlcxx_personality_v1", EHPersonality::XL_CXX) .Case("__zos_cxx_personality_v2", EHPersonality::ZOS_CXX) @@ -77,7 +78,8 @@ StringRef llvm::getEHPersonalityName(EHPersonality Pers) { case EHPersonality::CoreCLR: return "ProcessCLRException"; case EHPersonality::Rust: - return "rust_eh_personality"; + llvm_unreachable( + "Cannot get personality name of Rust personality, since it is mangled"); case EHPersonality::Wasm_CXX: return "__gxx_wasm_personality_v0"; case EHPersonality::XL_CXX: diff --git a/llvm/lib/IR/FPEnv.cpp b/llvm/lib/IR/FPEnv.cpp index 67f21d3..c41d7b3 100644 --- a/llvm/lib/IR/FPEnv.cpp +++ b/llvm/lib/IR/FPEnv.cpp @@ -19,9 +19,10 @@ #include "llvm/IR/Intrinsics.h" #include <optional> -namespace llvm { +using namespace llvm; -std::optional<RoundingMode> convertStrToRoundingMode(StringRef RoundingArg) { +std::optional<RoundingMode> +llvm::convertStrToRoundingMode(StringRef RoundingArg) { // For dynamic rounding mode, we use round to nearest but we will set the // 'exact' SDNodeFlag so that the value will not be rounded. return StringSwitch<std::optional<RoundingMode>>(RoundingArg) @@ -34,7 +35,8 @@ std::optional<RoundingMode> convertStrToRoundingMode(StringRef RoundingArg) { .Default(std::nullopt); } -std::optional<StringRef> convertRoundingModeToStr(RoundingMode UseRounding) { +std::optional<StringRef> +llvm::convertRoundingModeToStr(RoundingMode UseRounding) { std::optional<StringRef> RoundingStr; switch (UseRounding) { case RoundingMode::Dynamic: @@ -62,7 +64,7 @@ std::optional<StringRef> convertRoundingModeToStr(RoundingMode UseRounding) { } std::optional<fp::ExceptionBehavior> -convertStrToExceptionBehavior(StringRef ExceptionArg) { +llvm::convertStrToExceptionBehavior(StringRef ExceptionArg) { return StringSwitch<std::optional<fp::ExceptionBehavior>>(ExceptionArg) .Case("fpexcept.ignore", fp::ebIgnore) .Case("fpexcept.maytrap", fp::ebMayTrap) @@ -71,7 +73,7 @@ convertStrToExceptionBehavior(StringRef ExceptionArg) { } std::optional<StringRef> -convertExceptionBehaviorToStr(fp::ExceptionBehavior UseExcept) { +llvm::convertExceptionBehaviorToStr(fp::ExceptionBehavior UseExcept) { std::optional<StringRef> ExceptStr; switch (UseExcept) { case fp::ebStrict: @@ -87,7 +89,7 @@ convertExceptionBehaviorToStr(fp::ExceptionBehavior UseExcept) { return ExceptStr; } -Intrinsic::ID getConstrainedIntrinsicID(const Instruction &Instr) { +Intrinsic::ID llvm::getConstrainedIntrinsicID(const Instruction &Instr) { Intrinsic::ID IID = Intrinsic::not_intrinsic; switch (Instr.getOpcode()) { case Instruction::FCmp: @@ -127,5 +129,3 @@ Intrinsic::ID getConstrainedIntrinsicID(const Instruction &Instr) { return IID; } - -} // namespace llvm diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp index fc06745..31a2944 100644 --- a/llvm/lib/IR/Function.cpp +++ b/llvm/lib/IR/Function.cpp @@ -396,6 +396,9 @@ Function *Function::createWithDefaultAttr(FunctionType *Ty, case FramePointerKind::NonLeaf: B.addAttribute("frame-pointer", "non-leaf"); break; + case FramePointerKind::NonLeafNoReserve: + B.addAttribute("frame-pointer", "non-leaf-no-reserve"); + break; case FramePointerKind::All: B.addAttribute("frame-pointer", "all"); break; diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp index 88dbd17..8e1707a 100644 --- a/llvm/lib/IR/IRBuilder.cpp +++ b/llvm/lib/IR/IRBuilder.cpp @@ -858,24 +858,12 @@ CallInst *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID, const Twine &Name) { Module *M = BB->getModule(); - SmallVector<Intrinsic::IITDescriptor> Table; - Intrinsic::getIntrinsicInfoTableEntries(ID, Table); - ArrayRef<Intrinsic::IITDescriptor> TableRef(Table); - SmallVector<Type *> ArgTys; ArgTys.reserve(Args.size()); for (auto &I : Args) ArgTys.push_back(I->getType()); - FunctionType *FTy = FunctionType::get(RetTy, ArgTys, false); - SmallVector<Type *> OverloadTys; - Intrinsic::MatchIntrinsicTypesResult Res = - matchIntrinsicSignature(FTy, TableRef, OverloadTys); - (void)Res; - assert(Res == Intrinsic::MatchIntrinsicTypes_Match && TableRef.empty() && - "Wrong types for intrinsic!"); - // TODO: Handle varargs intrinsics. - - Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, OverloadTys); + + Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, RetTy, ArgTys); return createCallHelper(Fn, Args, Name, FMFSource); } @@ -1019,8 +1007,7 @@ Value *IRBuilderBase::CreateSelectWithUnknownProfile(Value *C, Value *True, const Twine &Name) { Value *Ret = CreateSelectFMF(C, True, False, {}, Name); if (auto *SI = dyn_cast<SelectInst>(Ret)) { - setExplicitlyUnknownBranchWeightsIfProfiled( - *SI, *SI->getParent()->getParent(), PassName); + setExplicitlyUnknownBranchWeightsIfProfiled(*SI, PassName); } return Ret; } diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index 3b8fde8..85d3690 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -620,7 +620,8 @@ bool CallBase::hasReadingOperandBundles() const { // ptrauth) forces a callsite to be at least readonly. return hasOperandBundlesOtherThan({LLVMContext::OB_ptrauth, LLVMContext::OB_kcfi, - LLVMContext::OB_convergencectrl}) && + LLVMContext::OB_convergencectrl, + LLVMContext::OB_deactivation_symbol}) && getIntrinsicID() != Intrinsic::assume; } @@ -628,7 +629,8 @@ bool CallBase::hasClobberingOperandBundles() const { return hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_ptrauth, LLVMContext::OB_kcfi, - LLVMContext::OB_convergencectrl}) && + LLVMContext::OB_convergencectrl, + LLVMContext::OB_deactivation_symbol}) && getIntrinsicID() != Intrinsic::assume; } @@ -4171,6 +4173,16 @@ SwitchInstProfUpdateWrapper::removeCase(SwitchInst::CaseIt I) { return SI.removeCase(I); } +void SwitchInstProfUpdateWrapper::replaceDefaultDest(SwitchInst::CaseIt I) { + auto *DestBlock = I->getCaseSuccessor(); + if (Weights) { + auto Weight = getSuccessorWeight(I->getCaseIndex() + 1); + (*Weights)[0] = Weight.value(); + } + + SI.setDefaultDest(DestBlock); +} + void SwitchInstProfUpdateWrapper::addCase( ConstantInt *OnVal, BasicBlock *Dest, SwitchInstProfUpdateWrapper::CaseWeightOpt W) { diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp index 23a4d1b..281cbd4 100644 --- a/llvm/lib/IR/IntrinsicInst.cpp +++ b/llvm/lib/IR/IntrinsicInst.cpp @@ -697,9 +697,6 @@ Function *VPIntrinsic::getOrInsertDeclarationForParams( VPFunc = Intrinsic::getOrInsertDeclaration( M, VPID, {Params[0]->getType(), Params[1]->getType()}); break; - case Intrinsic::experimental_vp_splat: - VPFunc = Intrinsic::getOrInsertDeclaration(M, VPID, ReturnType); - break; } assert(VPFunc && "Could not declare VP intrinsic"); return VPFunc; diff --git a/llvm/lib/IR/Intrinsics.cpp b/llvm/lib/IR/Intrinsics.cpp index 526800e..f46d3e5 100644 --- a/llvm/lib/IR/Intrinsics.cpp +++ b/llvm/lib/IR/Intrinsics.cpp @@ -32,6 +32,7 @@ #include "llvm/IR/IntrinsicsX86.h" #include "llvm/IR/IntrinsicsXCore.h" #include "llvm/IR/Module.h" +#include "llvm/IR/NVVMIntrinsicUtils.h" #include "llvm/IR/Type.h" using namespace llvm; @@ -601,6 +602,12 @@ bool Intrinsic::isOverloaded(ID id) { #undef GET_INTRINSIC_OVERLOAD_TABLE } +bool Intrinsic::hasPrettyPrintedArgs(ID id){ +#define GET_INTRINSIC_PRETTY_PRINT_TABLE +#include "llvm/IR/IntrinsicImpl.inc" +#undef GET_INTRINSIC_PRETTY_PRINT_TABLE +} + /// Table of per-target intrinsic name tables. #define GET_INTRINSIC_TARGET_DATA #include "llvm/IR/IntrinsicImpl.inc" @@ -720,14 +727,14 @@ Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) { #include "llvm/IR/IntrinsicImpl.inc" #undef GET_INTRINSIC_ATTRIBUTES -Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id, - ArrayRef<Type *> Tys) { - // There can never be multiple globals with the same name of different types, - // because intrinsics must be a specific type. - auto *FT = getType(M->getContext(), id, Tys); +static Function *getOrInsertIntrinsicDeclarationImpl(Module *M, + Intrinsic::ID id, + ArrayRef<Type *> Tys, + FunctionType *FT) { Function *F = cast<Function>( - M->getOrInsertFunction( - Tys.empty() ? getName(id) : getName(id, Tys, M, FT), FT) + M->getOrInsertFunction(Tys.empty() ? Intrinsic::getName(id) + : Intrinsic::getName(id, Tys, M, FT), + FT) .getCallee()); if (F->getFunctionType() == FT) return F; @@ -739,11 +746,49 @@ Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id, // invalid declaration will get upgraded later. F->setName(F->getName() + ".invalid"); return cast<Function>( - M->getOrInsertFunction( - Tys.empty() ? getName(id) : getName(id, Tys, M, FT), FT) + M->getOrInsertFunction(Tys.empty() ? Intrinsic::getName(id) + : Intrinsic::getName(id, Tys, M, FT), + FT) .getCallee()); } +Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id, + ArrayRef<Type *> Tys) { + // There can never be multiple globals with the same name of different types, + // because intrinsics must be a specific type. + FunctionType *FT = getType(M->getContext(), id, Tys); + return getOrInsertIntrinsicDeclarationImpl(M, id, Tys, FT); +} + +Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id, Type *RetTy, + ArrayRef<Type *> ArgTys) { + // If the intrinsic is not overloaded, use the non-overloaded version. + if (!Intrinsic::isOverloaded(id)) + return getOrInsertDeclaration(M, id); + + // Get the intrinsic signature metadata. + SmallVector<Intrinsic::IITDescriptor, 8> Table; + getIntrinsicInfoTableEntries(id, Table); + ArrayRef<Intrinsic::IITDescriptor> TableRef = Table; + + FunctionType *FTy = FunctionType::get(RetTy, ArgTys, /*isVarArg=*/false); + + // Automatically determine the overloaded types. + SmallVector<Type *, 4> OverloadTys; + [[maybe_unused]] Intrinsic::MatchIntrinsicTypesResult Res = + matchIntrinsicSignature(FTy, TableRef, OverloadTys); + assert(Res == Intrinsic::MatchIntrinsicTypes_Match && + "intrinsic signature mismatch"); + + // If intrinsic requires vararg, recreate the FunctionType accordingly. + if (!matchIntrinsicVarArg(/*isVarArg=*/true, TableRef)) + FTy = FunctionType::get(RetTy, ArgTys, /*isVarArg=*/true); + + assert(TableRef.empty() && "Unprocessed descriptors remain"); + + return getOrInsertIntrinsicDeclarationImpl(M, id, OverloadTys, FTy); +} + Function *Intrinsic::getDeclarationIfExists(const Module *M, ID id) { return M->getFunction(getName(id)); } @@ -1142,3 +1187,7 @@ Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) { assert(Factor >= 2 && Factor <= 8 && "Unexpected factor"); return InterleaveIntrinsics[Factor - 2].Deinterleave; } + +#define GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS +#include "llvm/IR/IntrinsicImpl.inc" +#undef GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp index 335c210..10aba75 100644 --- a/llvm/lib/IR/LLVMContext.cpp +++ b/llvm/lib/IR/LLVMContext.cpp @@ -55,6 +55,8 @@ static StringRef knownBundleName(unsigned BundleTagID) { return "convergencectrl"; case LLVMContext::OB_align: return "align"; + case LLVMContext::OB_deactivation_symbol: + return "deactivation-symbol"; default: llvm_unreachable("unknown bundle id"); } diff --git a/llvm/lib/IR/LLVMContextImpl.cpp b/llvm/lib/IR/LLVMContextImpl.cpp index 87037c3..8f79398 100644 --- a/llvm/lib/IR/LLVMContextImpl.cpp +++ b/llvm/lib/IR/LLVMContextImpl.cpp @@ -25,7 +25,6 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" #include <cassert> -#include <utility> using namespace llvm; @@ -108,6 +107,7 @@ LLVMContextImpl::~LLVMContextImpl() { ArrayConstants.freeConstants(); StructConstants.freeConstants(); VectorConstants.freeConstants(); + ConstantPtrAuths.freeConstants(); InlineAsms.freeConstants(); CAZConstants.clear(); diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp index 62fd62c..3394754 100644 --- a/llvm/lib/IR/ModuleSummaryIndex.cpp +++ b/llvm/lib/IR/ModuleSummaryIndex.cpp @@ -34,8 +34,6 @@ static cl::opt<bool> ImportConstantsWithRefs( "import-constants-with-refs", cl::init(true), cl::Hidden, cl::desc("Import constant global variables with references")); -constexpr uint32_t FunctionSummary::ParamAccess::RangeWidth; - FunctionSummary FunctionSummary::ExternalNode = FunctionSummary::makeDummyFunctionSummary( SmallVector<FunctionSummary::EdgeTy, 0>()); @@ -88,8 +86,6 @@ std::pair<unsigned, unsigned> FunctionSummary::specialRefCounts() const { return {RORefCnt, WORefCnt}; } -constexpr uint64_t ModuleSummaryIndex::BitcodeSummaryVersion; - uint64_t ModuleSummaryIndex::getFlags() const { uint64_t Flags = 0; // Flags & 0x4 is reserved. DO NOT REUSE. diff --git a/llvm/lib/IR/NVVMIntrinsicUtils.cpp b/llvm/lib/IR/NVVMIntrinsicUtils.cpp new file mode 100644 index 0000000..4389fa3 --- /dev/null +++ b/llvm/lib/IR/NVVMIntrinsicUtils.cpp @@ -0,0 +1,61 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements functions associated with NVVM Intrinsics. +// +//===----------------------------------------------------------------------===// + +#include "llvm/IR/NVVMIntrinsicUtils.h" + +using namespace llvm; +using namespace nvvm; + +void nvvm::printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal) { + if (const auto *CI = dyn_cast<ConstantInt>(ImmArgVal)) { + uint64_t Val = CI->getZExtValue(); + switch (static_cast<Tcgen05MMAKind>(Val)) { + case Tcgen05MMAKind::F16: + OS << "f16"; + return; + case Tcgen05MMAKind::TF32: + OS << "tf32"; + return; + case Tcgen05MMAKind::F8F6F4: + OS << "f8f6f4"; + return; + case Tcgen05MMAKind::I8: + OS << "i8"; + return; + } + } + llvm_unreachable( + "printTcgen05MMAKind called with invalid value for immediate argument"); +} + +void nvvm::printTcgen05CollectorUsageOp(raw_ostream &OS, + const Constant *ImmArgVal) { + if (const auto *CI = dyn_cast<ConstantInt>(ImmArgVal)) { + uint64_t Val = CI->getZExtValue(); + switch (static_cast<Tcgen05CollectorUsageOp>(Val)) { + case Tcgen05CollectorUsageOp::DISCARD: + OS << "discard"; + return; + case Tcgen05CollectorUsageOp::LASTUSE: + OS << "lastuse"; + return; + case Tcgen05CollectorUsageOp::FILL: + OS << "fill"; + return; + case Tcgen05CollectorUsageOp::USE: + OS << "use"; + return; + } + } + llvm_unreachable("printTcgen05CollectorUsageOp called with invalid value for " + "immediate argument"); +} diff --git a/llvm/lib/IR/Operator.cpp b/llvm/lib/IR/Operator.cpp index 39e5463c..c3e54a0 100644 --- a/llvm/lib/IR/Operator.cpp +++ b/llvm/lib/IR/Operator.cpp @@ -17,7 +17,8 @@ #include "ConstantsContext.h" -namespace llvm { +using namespace llvm; + bool Operator::hasPoisonGeneratingFlags() const { switch (getOpcode()) { case Instruction::Add: @@ -288,4 +289,3 @@ void FastMathFlags::print(raw_ostream &O) const { O << " afn"; } } -} // namespace llvm diff --git a/llvm/lib/IR/PassRegistry.cpp b/llvm/lib/IR/PassRegistry.cpp index 94afbb5..a91bb56 100644 --- a/llvm/lib/IR/PassRegistry.cpp +++ b/llvm/lib/IR/PassRegistry.cpp @@ -17,7 +17,6 @@ #include "llvm/PassInfo.h" #include <cassert> #include <memory> -#include <utility> using namespace llvm; diff --git a/llvm/lib/IR/PassTimingInfo.cpp b/llvm/lib/IR/PassTimingInfo.cpp index 4e27086..cb1b91a 100644 --- a/llvm/lib/IR/PassTimingInfo.cpp +++ b/llvm/lib/IR/PassTimingInfo.cpp @@ -32,10 +32,10 @@ using namespace llvm; #define DEBUG_TYPE "time-passes" -namespace llvm { +using namespace llvm; -bool TimePassesIsEnabled = false; -bool TimePassesPerRun = false; +bool llvm::TimePassesIsEnabled = false; +bool llvm::TimePassesPerRun = false; static cl::opt<bool, true> EnableTiming( "time-passes", cl::location(TimePassesIsEnabled), cl::Hidden, @@ -139,7 +139,7 @@ PassTimingInfo *PassTimingInfo::TheTimeInfo; } // namespace legacy } // namespace -Timer *getPassTimer(Pass *P) { +Timer *llvm::getPassTimer(Pass *P) { legacy::PassTimingInfo::init(); if (legacy::PassTimingInfo::TheTimeInfo) return legacy::PassTimingInfo::TheTimeInfo->getPassTimer(P, P); @@ -148,7 +148,7 @@ Timer *getPassTimer(Pass *P) { /// If timing is enabled, report the times collected up to now and then reset /// them. -void reportAndResetTimings(raw_ostream *OutStream) { +void llvm::reportAndResetTimings(raw_ostream *OutStream) { if (legacy::PassTimingInfo::TheTimeInfo) legacy::PassTimingInfo::TheTimeInfo->print(OutStream); } @@ -315,5 +315,3 @@ void TimePassesHandler::registerCallbacks(PassInstrumentationCallbacks &PIC) { PIC.registerAfterAnalysisCallback( [this](StringRef P, Any) { this->stopAnalysisTimer(P); }); } - -} // namespace llvm diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp index fc2be51..d1ada00 100644 --- a/llvm/lib/IR/ProfDataUtils.cpp +++ b/llvm/lib/IR/ProfDataUtils.cpp @@ -86,7 +86,7 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData, } /// Push the weights right to fit in uint32_t. -static SmallVector<uint32_t> fitWeights(ArrayRef<uint64_t> Weights) { +SmallVector<uint32_t> llvm::fitWeights(ArrayRef<uint64_t> Weights) { SmallVector<uint32_t> Ret; Ret.reserve(Weights.size()); uint64_t Max = *llvm::max_element(Weights); @@ -274,9 +274,12 @@ void llvm::setExplicitlyUnknownBranchWeights(Instruction &I, } void llvm::setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, - Function &F, - StringRef PassName) { - if (std::optional<Function::ProfileCount> EC = F.getEntryCount(); + StringRef PassName, + const Function *F) { + F = F ? F : I.getFunction(); + assert(F && "Either pass a instruction attached to a Function, or explicitly " + "pass the Function that it will be attached to"); + if (std::optional<Function::ProfileCount> EC = F->getEntryCount(); EC && EC->getCount() > 0) setExplicitlyUnknownBranchWeights(I, PassName); } diff --git a/llvm/lib/IR/PseudoProbe.cpp b/llvm/lib/IR/PseudoProbe.cpp index 59f218c..3c05f4b 100644 --- a/llvm/lib/IR/PseudoProbe.cpp +++ b/llvm/lib/IR/PseudoProbe.cpp @@ -19,9 +19,7 @@ using namespace llvm; -namespace llvm { - -std::optional<PseudoProbe> +static std::optional<PseudoProbe> extractProbeFromDiscriminator(const DILocation *DIL) { if (DIL) { auto Discriminator = DIL->getDiscriminator(); @@ -43,7 +41,7 @@ extractProbeFromDiscriminator(const DILocation *DIL) { return std::nullopt; } -std::optional<PseudoProbe> +static std::optional<PseudoProbe> extractProbeFromDiscriminator(const Instruction &Inst) { assert(isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst) && "Only call instructions should have pseudo probe encodes as their " @@ -53,7 +51,7 @@ extractProbeFromDiscriminator(const Instruction &Inst) { return std::nullopt; } -std::optional<PseudoProbe> extractProbe(const Instruction &Inst) { +std::optional<PseudoProbe> llvm::extractProbe(const Instruction &Inst) { if (const auto *II = dyn_cast<PseudoProbeInst>(&Inst)) { PseudoProbe Probe; Probe.Id = II->getIndex()->getZExtValue(); @@ -73,7 +71,7 @@ std::optional<PseudoProbe> extractProbe(const Instruction &Inst) { return std::nullopt; } -void setProbeDistributionFactor(Instruction &Inst, float Factor) { +void llvm::setProbeDistributionFactor(Instruction &Inst, float Factor) { assert(Factor >= 0 && Factor <= 1 && "Distribution factor must be in [0, 1.0]"); if (auto *II = dyn_cast<PseudoProbeInst>(&Inst)) { @@ -111,5 +109,3 @@ void setProbeDistributionFactor(Instruction &Inst, float Factor) { } } } - -} // namespace llvm diff --git a/llvm/lib/IR/ReplaceConstant.cpp b/llvm/lib/IR/ReplaceConstant.cpp index 962368f..b1864c3 100644 --- a/llvm/lib/IR/ReplaceConstant.cpp +++ b/llvm/lib/IR/ReplaceConstant.cpp @@ -16,15 +16,15 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" -namespace llvm { +using namespace llvm; static bool isExpandableUser(User *U) { return isa<ConstantExpr>(U) || isa<ConstantAggregate>(U); } -static SmallVector<Instruction *, 4> expandUser(BasicBlock::iterator InsertPt, - Constant *C) { - SmallVector<Instruction *, 4> NewInsts; +static void expandUser(BasicBlock::iterator InsertPt, Constant *C, + SmallVector<Instruction *, 4> &NewInsts) { + NewInsts.clear(); if (auto *CE = dyn_cast<ConstantExpr>(C)) { Instruction *ConstInst = CE->getAsInstruction(); ConstInst->insertBefore(*InsertPt->getParent(), InsertPt); @@ -46,13 +46,12 @@ static SmallVector<Instruction *, 4> expandUser(BasicBlock::iterator InsertPt, } else { llvm_unreachable("Not an expandable user"); } - return NewInsts; } -bool convertUsersOfConstantsToInstructions(ArrayRef<Constant *> Consts, - Function *RestrictToFunc, - bool RemoveDeadConstants, - bool IncludeSelf) { +bool llvm::convertUsersOfConstantsToInstructions(ArrayRef<Constant *> Consts, + Function *RestrictToFunc, + bool RemoveDeadConstants, + bool IncludeSelf) { // Find all expandable direct users of Consts. SmallVector<Constant *> Stack; for (Constant *C : Consts) { @@ -91,6 +90,11 @@ bool convertUsersOfConstantsToInstructions(ArrayRef<Constant *> Consts, // Replace those expandable operands with instructions bool Changed = false; + // We need to cache the instructions we've already expanded to avoid expanding + // the same constant multiple times in the same basic block, which is + // problematic when the same constant is used in a phi node multiple times. + DenseMap<std::pair<Constant *, BasicBlock *>, SmallVector<Instruction *, 4>> + ConstantToInstructionMap; while (!InstructionWorklist.empty()) { Instruction *I = InstructionWorklist.pop_back_val(); DebugLoc Loc = I->getDebugLoc(); @@ -105,7 +109,14 @@ bool convertUsersOfConstantsToInstructions(ArrayRef<Constant *> Consts, if (auto *C = dyn_cast<Constant>(U.get())) { if (ExpandableUsers.contains(C)) { Changed = true; - auto NewInsts = expandUser(BI, C); + SmallVector<Instruction *, 4> &NewInsts = + ConstantToInstructionMap[std::make_pair(C, BI->getParent())]; + // If the cached instruction is after the insertion point, we need to + // create a new one. We can't simply move the cached instruction + // because its operands (also expanded instructions) might not + // dominate the new position. + if (NewInsts.empty() || BI->comesBefore(NewInsts.front())) + expandUser(BI, C, NewInsts); for (auto *NI : NewInsts) NI->setDebugLoc(Loc); InstructionWorklist.insert_range(NewInsts); @@ -121,5 +132,3 @@ bool convertUsersOfConstantsToInstructions(ArrayRef<Constant *> Consts, return Changed; } - -} // namespace llvm diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp index 77af29b..a5f842a 100644 --- a/llvm/lib/IR/RuntimeLibcalls.cpp +++ b/llvm/lib/IR/RuntimeLibcalls.cpp @@ -7,7 +7,10 @@ //===----------------------------------------------------------------------===// #include "llvm/IR/RuntimeLibcalls.h" +#include "llvm/ADT/FloatingPointMode.h" #include "llvm/ADT/StringTable.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/SystemLibraries.h" #include "llvm/Support/Debug.h" #include "llvm/Support/xxhash.h" #include "llvm/TargetParser/ARMTargetParser.h" @@ -17,11 +20,65 @@ using namespace llvm; using namespace RTLIB; +#define GET_RUNTIME_LIBCALLS_INFO #define GET_INIT_RUNTIME_LIBCALL_NAMES #define GET_SET_TARGET_RUNTIME_LIBCALL_SETS #define DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME #include "llvm/IR/RuntimeLibcalls.inc" +RuntimeLibcallsInfo::RuntimeLibcallsInfo(const Triple &TT, + ExceptionHandling ExceptionModel, + FloatABI::ABIType FloatABI, + EABI EABIVersion, StringRef ABIName, + VectorLibrary VecLib) { + // FIXME: The ExceptionModel parameter is to handle the field in + // TargetOptions. This interface fails to distinguish the forced disable + // case for targets which support exceptions by default. This should + // probably be a module flag and removed from TargetOptions. + if (ExceptionModel == ExceptionHandling::None) + ExceptionModel = TT.getDefaultExceptionHandling(); + + initLibcalls(TT, ExceptionModel, FloatABI, EABIVersion, ABIName); + + // TODO: Tablegen should generate these sets + switch (VecLib) { + case VectorLibrary::SLEEFGNUABI: + for (RTLIB::LibcallImpl Impl : + {RTLIB::impl__ZGVnN2vl8_modf, RTLIB::impl__ZGVnN4vl4_modff, + RTLIB::impl__ZGVsNxvl8_modf, RTLIB::impl__ZGVsNxvl4_modff, + RTLIB::impl__ZGVnN2vl8l8_sincos, RTLIB::impl__ZGVnN4vl4l4_sincosf, + RTLIB::impl__ZGVsNxvl8l8_sincos, RTLIB::impl__ZGVsNxvl4l4_sincosf, + RTLIB::impl__ZGVnN4vl4l4_sincospif, RTLIB::impl__ZGVnN2vl8l8_sincospi, + RTLIB::impl__ZGVsNxvl4l4_sincospif, + RTLIB::impl__ZGVsNxvl8l8_sincospi}) + setAvailable(Impl); + break; + case VectorLibrary::ArmPL: + for (RTLIB::LibcallImpl Impl : + {RTLIB::impl_armpl_vmodfq_f64, RTLIB::impl_armpl_vmodfq_f32, + RTLIB::impl_armpl_svmodf_f64_x, RTLIB::impl_armpl_svmodf_f32_x, + RTLIB::impl_armpl_vsincosq_f64, RTLIB::impl_armpl_vsincosq_f32, + RTLIB::impl_armpl_svsincos_f64_x, RTLIB::impl_armpl_svsincos_f32_x, + RTLIB::impl_armpl_vsincospiq_f32, RTLIB::impl_armpl_vsincospiq_f64, + RTLIB::impl_armpl_svsincospi_f32_x, + RTLIB::impl_armpl_svsincospi_f64_x}) + setAvailable(Impl); + + for (RTLIB::LibcallImpl Impl : + {RTLIB::impl_armpl_vsincosq_f64, RTLIB::impl_armpl_vsincosq_f32}) + setLibcallImplCallingConv(Impl, CallingConv::AArch64_VectorCall); + + break; + default: + break; + } +} + +RuntimeLibcallsInfo::RuntimeLibcallsInfo(const Module &M) + : RuntimeLibcallsInfo(M.getTargetTriple()) { + // TODO: Consider module flags +} + /// Set default libcall names. If a target wants to opt-out of a libcall it /// should be placed here. void RuntimeLibcallsInfo::initLibcalls(const Triple &TT, @@ -72,3 +129,282 @@ bool RuntimeLibcallsInfo::darwinHasExp10(const Triple &TT) { return false; } } + +/// TODO: There is really no guarantee that sizeof(size_t) is equal to the index +/// size of the default address space. This matches TargetLibraryInfo and should +/// be kept in sync. +static IntegerType *getSizeTType(LLVMContext &Ctx, const DataLayout &DL) { + return DL.getIndexType(Ctx, /*AddressSpace=*/0); +} + +std::pair<FunctionType *, AttributeList> +RuntimeLibcallsInfo::getFunctionTy(LLVMContext &Ctx, const Triple &TT, + const DataLayout &DL, + RTLIB::LibcallImpl LibcallImpl) const { + // TODO: NoCallback probably unsafe in general + static constexpr Attribute::AttrKind CommonFnAttrs[] = { + Attribute::MustProgress, Attribute::NoCallback, Attribute::NoFree, + Attribute::NoSync, Attribute::NoUnwind, Attribute::WillReturn}; + static constexpr Attribute::AttrKind MemoryFnAttrs[] = { + Attribute::MustProgress, Attribute::NoUnwind, Attribute::WillReturn}; + static constexpr Attribute::AttrKind CommonPtrArgAttrs[] = { + Attribute::NoAlias, Attribute::WriteOnly, Attribute::NonNull}; + + switch (LibcallImpl) { + case RTLIB::impl___sincos_stret: + case RTLIB::impl___sincosf_stret: { + if (!darwinHasSinCosStret(TT)) // Non-darwin currently unexpected + return {}; + + Type *ScalarTy = LibcallImpl == RTLIB::impl___sincosf_stret + ? Type::getFloatTy(Ctx) + : Type::getDoubleTy(Ctx); + + AttrBuilder FuncAttrBuilder(Ctx); + for (Attribute::AttrKind Attr : CommonFnAttrs) + FuncAttrBuilder.addAttribute(Attr); + + const bool UseSret = + TT.isX86_32() || ((TT.isARM() || TT.isThumb()) && + ARM::computeTargetABI(TT) == ARM::ARM_ABI_APCS); + + FuncAttrBuilder.addMemoryAttr(MemoryEffects::argumentOrErrnoMemOnly( + UseSret ? ModRefInfo::Mod : ModRefInfo::NoModRef, ModRefInfo::Mod)); + + AttributeList Attrs; + Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder); + + if (UseSret) { + AttrBuilder AttrBuilder(Ctx); + StructType *StructTy = StructType::get(ScalarTy, ScalarTy); + AttrBuilder.addStructRetAttr(StructTy); + AttrBuilder.addAlignmentAttr(DL.getABITypeAlign(StructTy)); + FunctionType *FuncTy = FunctionType::get( + Type::getVoidTy(Ctx), {DL.getAllocaPtrType(Ctx), ScalarTy}, false); + + return {FuncTy, Attrs.addParamAttributes(Ctx, 0, AttrBuilder)}; + } + + Type *RetTy = + LibcallImpl == RTLIB::impl___sincosf_stret && TT.isX86_64() + ? static_cast<Type *>(FixedVectorType::get(ScalarTy, 2)) + : static_cast<Type *>(StructType::get(ScalarTy, ScalarTy)); + + return {FunctionType::get(RetTy, {ScalarTy}, false), Attrs}; + } + case RTLIB::impl_malloc: + case RTLIB::impl_calloc: { + AttrBuilder FuncAttrBuilder(Ctx); + for (Attribute::AttrKind Attr : MemoryFnAttrs) + FuncAttrBuilder.addAttribute(Attr); + FuncAttrBuilder.addAttribute(Attribute::NoFree); + + AllocFnKind AllocKind = AllocFnKind::Alloc; + if (LibcallImpl == RTLIB::impl_malloc) + AllocKind |= AllocFnKind::Uninitialized; + + // TODO: Set memory attribute + FuncAttrBuilder.addAllocKindAttr(AllocKind); + FuncAttrBuilder.addAttribute("alloc-family", "malloc"); + FuncAttrBuilder.addAllocSizeAttr(0, LibcallImpl == RTLIB::impl_malloc + ? std::nullopt + : std::make_optional(1)); + + AttributeList Attrs; + Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder); + + { + AttrBuilder ArgAttrBuilder(Ctx); + for (Attribute::AttrKind AK : CommonPtrArgAttrs) + ArgAttrBuilder.addAttribute(AK); + + Attrs = Attrs.addRetAttribute(Ctx, Attribute::NoUndef); + Attrs = Attrs.addRetAttribute(Ctx, Attribute::NoAlias); + Attrs = Attrs.addParamAttribute(Ctx, 0, Attribute::NoUndef); + if (LibcallImpl == RTLIB::impl_calloc) + Attrs = Attrs.addParamAttribute(Ctx, 1, Attribute::NoUndef); + } + + IntegerType *SizeT = getSizeTType(Ctx, DL); + PointerType *PtrTy = PointerType::get(Ctx, 0); + SmallVector<Type *, 2> ArgTys = {SizeT}; + if (LibcallImpl == RTLIB::impl_calloc) + ArgTys.push_back(SizeT); + + return {FunctionType::get(PtrTy, ArgTys, false), Attrs}; + } + case RTLIB::impl_free: { + // TODO: Set memory attribute + AttrBuilder FuncAttrBuilder(Ctx); + for (Attribute::AttrKind Attr : MemoryFnAttrs) + FuncAttrBuilder.addAttribute(Attr); + + FuncAttrBuilder.addAllocKindAttr(AllocFnKind::Free); + FuncAttrBuilder.addAttribute("alloc-family", "malloc"); + + AttributeList Attrs; + Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder); + + { + AttrBuilder ArgAttrBuilder(Ctx); + ArgAttrBuilder.addAttribute(Attribute::NoUndef); + ArgAttrBuilder.addAttribute(Attribute::AllocatedPointer); + ArgAttrBuilder.addCapturesAttr(CaptureInfo::none()); + Attrs = Attrs.addParamAttributes(Ctx, 0, ArgAttrBuilder); + } + + return {FunctionType::get(Type::getVoidTy(Ctx), {PointerType::get(Ctx, 0)}, + false), + Attrs}; + } + case RTLIB::impl_sqrtf: + case RTLIB::impl_sqrt: { + AttrBuilder FuncAttrBuilder(Ctx); + + for (Attribute::AttrKind Attr : CommonFnAttrs) + FuncAttrBuilder.addAttribute(Attr); + FuncAttrBuilder.addMemoryAttr(MemoryEffects::errnoMemOnly(ModRefInfo::Mod)); + + AttributeList Attrs; + Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder); + + Type *ScalarTy = LibcallImpl == RTLIB::impl_sqrtf ? Type::getFloatTy(Ctx) + : Type::getDoubleTy(Ctx); + FunctionType *FuncTy = FunctionType::get(ScalarTy, {ScalarTy}, false); + + Attrs = Attrs.addRetAttribute( + Ctx, Attribute::getWithNoFPClass(Ctx, fcNegInf | fcNegSubnormal | + fcNegNormal)); + return {FuncTy, Attrs}; + } + case RTLIB::impl__ZGVnN2vl8_modf: + case RTLIB::impl__ZGVnN4vl4_modff: + case RTLIB::impl__ZGVsNxvl8_modf: + case RTLIB::impl__ZGVsNxvl4_modff: + case RTLIB::impl_armpl_vmodfq_f64: + case RTLIB::impl_armpl_vmodfq_f32: + case RTLIB::impl_armpl_svmodf_f64_x: + case RTLIB::impl_armpl_svmodf_f32_x: { + AttrBuilder FuncAttrBuilder(Ctx); + + bool IsF32 = LibcallImpl == RTLIB::impl__ZGVnN4vl4_modff || + LibcallImpl == RTLIB::impl__ZGVsNxvl4_modff || + LibcallImpl == RTLIB::impl_armpl_vmodfq_f32 || + LibcallImpl == RTLIB::impl_armpl_svmodf_f32_x; + + bool IsScalable = LibcallImpl == RTLIB::impl__ZGVsNxvl8_modf || + LibcallImpl == RTLIB::impl__ZGVsNxvl4_modff || + LibcallImpl == RTLIB::impl_armpl_svmodf_f64_x || + LibcallImpl == RTLIB::impl_armpl_svmodf_f32_x; + + Type *ScalarTy = IsF32 ? Type::getFloatTy(Ctx) : Type::getDoubleTy(Ctx); + unsigned EC = IsF32 ? 4 : 2; + VectorType *VecTy = VectorType::get(ScalarTy, EC, IsScalable); + + for (Attribute::AttrKind Attr : CommonFnAttrs) + FuncAttrBuilder.addAttribute(Attr); + FuncAttrBuilder.addMemoryAttr(MemoryEffects::argMemOnly(ModRefInfo::Mod)); + + AttributeList Attrs; + Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder); + + { + AttrBuilder ArgAttrBuilder(Ctx); + for (Attribute::AttrKind AK : CommonPtrArgAttrs) + ArgAttrBuilder.addAttribute(AK); + ArgAttrBuilder.addAlignmentAttr(DL.getABITypeAlign(VecTy)); + Attrs = Attrs.addParamAttributes(Ctx, 1, ArgAttrBuilder); + } + + PointerType *PtrTy = PointerType::get(Ctx, 0); + SmallVector<Type *, 4> ArgTys = {VecTy, PtrTy}; + if (hasVectorMaskArgument(LibcallImpl)) + ArgTys.push_back(VectorType::get(Type::getInt1Ty(Ctx), EC, IsScalable)); + + return {FunctionType::get(VecTy, ArgTys, false), Attrs}; + } + case RTLIB::impl__ZGVnN2vl8l8_sincos: + case RTLIB::impl__ZGVnN4vl4l4_sincosf: + case RTLIB::impl__ZGVsNxvl8l8_sincos: + case RTLIB::impl__ZGVsNxvl4l4_sincosf: + case RTLIB::impl_armpl_vsincosq_f64: + case RTLIB::impl_armpl_vsincosq_f32: + case RTLIB::impl_armpl_svsincos_f64_x: + case RTLIB::impl_armpl_svsincos_f32_x: + case RTLIB::impl__ZGVnN4vl4l4_sincospif: + case RTLIB::impl__ZGVnN2vl8l8_sincospi: + case RTLIB::impl__ZGVsNxvl4l4_sincospif: + case RTLIB::impl__ZGVsNxvl8l8_sincospi: + case RTLIB::impl_armpl_vsincospiq_f32: + case RTLIB::impl_armpl_vsincospiq_f64: + case RTLIB::impl_armpl_svsincospi_f32_x: + case RTLIB::impl_armpl_svsincospi_f64_x: { + AttrBuilder FuncAttrBuilder(Ctx); + + bool IsF32 = LibcallImpl == RTLIB::impl__ZGVnN4vl4l4_sincospif || + LibcallImpl == RTLIB::impl__ZGVsNxvl4l4_sincospif || + LibcallImpl == RTLIB::impl_armpl_vsincospiq_f32 || + LibcallImpl == RTLIB::impl_armpl_svsincospi_f32_x || + LibcallImpl == RTLIB::impl__ZGVnN4vl4l4_sincosf || + LibcallImpl == RTLIB::impl__ZGVsNxvl4l4_sincosf || + LibcallImpl == RTLIB::impl_armpl_vsincosq_f32 || + LibcallImpl == RTLIB::impl_armpl_svsincos_f32_x; + + Type *ScalarTy = IsF32 ? Type::getFloatTy(Ctx) : Type::getDoubleTy(Ctx); + unsigned EC = IsF32 ? 4 : 2; + + bool IsScalable = LibcallImpl == RTLIB::impl__ZGVsNxvl8l8_sincos || + LibcallImpl == RTLIB::impl__ZGVsNxvl4l4_sincosf || + LibcallImpl == RTLIB::impl_armpl_svsincos_f32_x || + LibcallImpl == RTLIB::impl_armpl_svsincos_f64_x || + LibcallImpl == RTLIB::impl__ZGVsNxvl4l4_sincospif || + LibcallImpl == RTLIB::impl__ZGVsNxvl8l8_sincospi || + LibcallImpl == RTLIB::impl_armpl_svsincospi_f32_x || + LibcallImpl == RTLIB::impl_armpl_svsincospi_f64_x; + VectorType *VecTy = VectorType::get(ScalarTy, EC, IsScalable); + + for (Attribute::AttrKind Attr : CommonFnAttrs) + FuncAttrBuilder.addAttribute(Attr); + FuncAttrBuilder.addMemoryAttr(MemoryEffects::argMemOnly(ModRefInfo::Mod)); + + AttributeList Attrs; + Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder); + + { + AttrBuilder ArgAttrBuilder(Ctx); + for (Attribute::AttrKind AK : CommonPtrArgAttrs) + ArgAttrBuilder.addAttribute(AK); + ArgAttrBuilder.addAlignmentAttr(DL.getABITypeAlign(VecTy)); + Attrs = Attrs.addParamAttributes(Ctx, 1, ArgAttrBuilder); + Attrs = Attrs.addParamAttributes(Ctx, 2, ArgAttrBuilder); + } + + PointerType *PtrTy = PointerType::get(Ctx, 0); + SmallVector<Type *, 4> ArgTys = {VecTy, PtrTy, PtrTy}; + if (hasVectorMaskArgument(LibcallImpl)) + ArgTys.push_back(VectorType::get(Type::getInt1Ty(Ctx), EC, IsScalable)); + + return {FunctionType::get(Type::getVoidTy(Ctx), ArgTys, false), Attrs}; + } + default: + return {}; + } + + return {}; +} + +bool RuntimeLibcallsInfo::hasVectorMaskArgument(RTLIB::LibcallImpl Impl) { + /// FIXME: This should be generated by tablegen and support the argument at an + /// arbitrary position + switch (Impl) { + case RTLIB::impl_armpl_svmodf_f64_x: + case RTLIB::impl_armpl_svmodf_f32_x: + case RTLIB::impl_armpl_svsincos_f32_x: + case RTLIB::impl_armpl_svsincos_f64_x: + case RTLIB::impl_armpl_svsincospi_f32_x: + case RTLIB::impl_armpl_svsincospi_f64_x: + return true; + default: + return false; + } +} diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp index 0e9535d..299d07b 100644 --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -28,7 +28,6 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/TargetParser/RISCVTargetParser.h" #include <cassert> -#include <utility> using namespace llvm; @@ -1008,6 +1007,10 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) { } if (Name == "spirv.IntegralConstant" || Name == "spirv.Literal") return TargetTypeInfo(Type::getVoidTy(C)); + if (Name == "spirv.Padding") + return TargetTypeInfo( + ArrayType::get(Type::getInt8Ty(C), Ty->getIntParameter(0)), + TargetExtType::CanBeGlobal); if (Name.starts_with("spirv.")) return TargetTypeInfo(PointerType::get(C, 0), TargetExtType::HasZeroInit, TargetExtType::CanBeGlobal, diff --git a/llvm/lib/IR/Use.cpp b/llvm/lib/IR/Use.cpp index 67882ba..5042335 100644 --- a/llvm/lib/IR/Use.cpp +++ b/llvm/lib/IR/Use.cpp @@ -9,7 +9,7 @@ #include "llvm/IR/Use.h" #include "llvm/IR/User.h" -namespace llvm { +using namespace llvm; void Use::swap(Use &RHS) { if (Val == RHS.Val) @@ -42,5 +42,3 @@ void Use::zap(Use *Start, const Use *Stop, bool del) { if (del) ::operator delete(Start); } - -} // namespace llvm diff --git a/llvm/lib/IR/User.cpp b/llvm/lib/IR/User.cpp index ab44cb4..9bb7c12 100644 --- a/llvm/lib/IR/User.cpp +++ b/llvm/lib/IR/User.cpp @@ -11,8 +11,11 @@ #include "llvm/IR/GlobalValue.h" #include "llvm/IR/IntrinsicInst.h" +using namespace llvm; + namespace llvm { class BasicBlock; +} //===----------------------------------------------------------------------===// // User Class @@ -214,5 +217,3 @@ LLVM_NO_SANITIZE_MEMORY_ATTRIBUTE void User::operator delete(void *Usr) { ::operator delete(Storage); } } - -} // namespace llvm diff --git a/llvm/lib/IR/ValueSymbolTable.cpp b/llvm/lib/IR/ValueSymbolTable.cpp index cd1cee1..3bf52f6 100644 --- a/llvm/lib/IR/ValueSymbolTable.cpp +++ b/llvm/lib/IR/ValueSymbolTable.cpp @@ -23,7 +23,6 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/TargetParser/Triple.h" #include <cassert> -#include <utility> using namespace llvm; diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 7917712..10e6c45 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -136,9 +136,7 @@ static cl::opt<bool> VerifyNoAliasScopeDomination( cl::desc("Ensure that llvm.experimental.noalias.scope.decl for identical " "scopes are not dominating")); -namespace llvm { - -struct VerifierSupport { +struct llvm::VerifierSupport { raw_ostream *OS; const Module &M; ModuleSlotTracker MST; @@ -192,6 +190,9 @@ private: case DbgVariableRecord::LocationType::Declare: *OS << "declare"; break; + case DbgVariableRecord::LocationType::DeclareValue: + *OS << "declare_value"; + break; case DbgVariableRecord::LocationType::Assign: *OS << "assign"; break; @@ -318,8 +319,6 @@ public: } }; -} // namespace llvm - namespace { class Verifier : public InstVisitor<Verifier>, VerifierSupport { @@ -1162,6 +1161,7 @@ void Verifier::visitMetadataAsValue(const MetadataAsValue &MDV, Function *F) { static bool isType(const Metadata *MD) { return !MD || isa<DIType>(MD); } static bool isScope(const Metadata *MD) { return !MD || isa<DIScope>(MD); } static bool isDINode(const Metadata *MD) { return !MD || isa<DINode>(MD); } +static bool isMDTuple(const Metadata *MD) { return !MD || isa<MDTuple>(MD); } void Verifier::visitDILocation(const DILocation &N) { CheckDI(N.getRawScope() && isa<DILocalScope>(N.getRawScope()), @@ -1187,13 +1187,17 @@ void Verifier::visitDISubrangeType(const DISubrangeType &N) { CheckDI(!BaseType || isType(BaseType), "BaseType must be a type"); auto *LBound = N.getRawLowerBound(); CheckDI(!LBound || isa<ConstantAsMetadata>(LBound) || - isa<DIVariable>(LBound) || isa<DIExpression>(LBound), - "LowerBound must be signed constant or DIVariable or DIExpression", + isa<DIVariable>(LBound) || isa<DIExpression>(LBound) || + isa<DIDerivedType>(LBound), + "LowerBound must be signed constant or DIVariable or DIExpression or " + "DIDerivedType", &N); auto *UBound = N.getRawUpperBound(); CheckDI(!UBound || isa<ConstantAsMetadata>(UBound) || - isa<DIVariable>(UBound) || isa<DIExpression>(UBound), - "UpperBound must be signed constant or DIVariable or DIExpression", + isa<DIVariable>(UBound) || isa<DIExpression>(UBound) || + isa<DIDerivedType>(UBound), + "UpperBound must be signed constant or DIVariable or DIExpression or " + "DIDerivedType", &N); auto *Stride = N.getRawStride(); CheckDI(!Stride || isa<ConstantAsMetadata>(Stride) || @@ -1324,6 +1328,30 @@ void Verifier::visitDIDerivedType(const DIDerivedType &N) { if (N.getTag() == dwarf::DW_TAG_ptr_to_member_type) { CheckDI(isType(N.getRawExtraData()), "invalid pointer to member type", &N, N.getRawExtraData()); + } else if (N.getTag() == dwarf::DW_TAG_template_alias) { + CheckDI(isMDTuple(N.getRawExtraData()), "invalid template parameters", &N, + N.getRawExtraData()); + } else if (N.getTag() == dwarf::DW_TAG_inheritance || + N.getTag() == dwarf::DW_TAG_member || + N.getTag() == dwarf::DW_TAG_variable) { + auto *ExtraData = N.getRawExtraData(); + auto IsValidExtraData = [&]() { + if (ExtraData == nullptr) + return true; + if (isa<ConstantAsMetadata>(ExtraData) || isa<MDString>(ExtraData) || + isa<DIObjCProperty>(ExtraData)) + return true; + if (auto *Tuple = dyn_cast<MDTuple>(ExtraData)) { + if (Tuple->getNumOperands() != 1) + return false; + return isa_and_nonnull<ConstantAsMetadata>(Tuple->getOperand(0).get()); + } + return false; + }; + CheckDI(IsValidExtraData(), + "extraData must be ConstantAsMetadata, MDString, DIObjCProperty, " + "or MDTuple with single ConstantAsMetadata operand", + &N, ExtraData); } if (N.getTag() == dwarf::DW_TAG_set_type) { @@ -1563,11 +1591,27 @@ void Verifier::visitDISubprogram(const DISubprogram &N) { auto *Node = dyn_cast<MDTuple>(RawNode); CheckDI(Node, "invalid retained nodes list", &N, RawNode); for (Metadata *Op : Node->operands()) { - CheckDI(Op && (isa<DILocalVariable>(Op) || isa<DILabel>(Op) || - isa<DIImportedEntity>(Op)), + CheckDI(Op, "nullptr in retained nodes", &N, Node); + + auto True = [](const Metadata *) { return true; }; + auto False = [](const Metadata *) { return false; }; + bool IsTypeCorrect = + DISubprogram::visitRetainedNode<bool>(Op, True, True, True, False); + CheckDI(IsTypeCorrect, "invalid retained nodes, expected DILocalVariable, DILabel or " "DIImportedEntity", &N, Node, Op); + + auto *RetainedNode = cast<DINode>(Op); + auto *RetainedNodeScope = dyn_cast_or_null<DILocalScope>( + DISubprogram::getRawRetainedNodeScope(RetainedNode)); + CheckDI(RetainedNodeScope, + "invalid retained nodes, retained node is not local", &N, Node, + RetainedNode); + CheckDI( + RetainedNodeScope->getSubprogram() == &N, + "invalid retained nodes, retained node does not belong to subprogram", + &N, Node, RetainedNode, RetainedNodeScope); } } CheckDI(!hasConflictingReferenceFlags(N.getFlags()), @@ -2484,7 +2528,8 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, if (Attribute FPAttr = Attrs.getFnAttr("frame-pointer"); FPAttr.isValid()) { StringRef FP = FPAttr.getValueAsString(); - if (FP != "all" && FP != "non-leaf" && FP != "none" && FP != "reserved") + if (FP != "all" && FP != "non-leaf" && FP != "none" && FP != "reserved" && + FP != "non-leaf-no-reserve") CheckFailed("invalid value for 'frame-pointer' attribute: " + FP, V); } @@ -2554,6 +2599,20 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, CheckFailed("invalid value for 'denormal-fp-math-f32' attribute: " + S, V); } + + if (auto A = Attrs.getFnAttr("modular-format"); A.isValid()) { + StringRef S = A.getValueAsString(); + SmallVector<StringRef> Args; + S.split(Args, ','); + Check(Args.size() >= 5, + "modular-format attribute requires at least 5 arguments", V); + unsigned FirstArgIdx; + Check(!Args[2].getAsInteger(10, FirstArgIdx), + "modular-format attribute first arg index is not an integer", V); + unsigned UpperBound = FT->getNumParams() + (FT->isVarArg() ? 1 : 0); + Check(FirstArgIdx > 0 && FirstArgIdx <= UpperBound, + "modular-format attribute first arg index is out of bounds", V); + } } void Verifier::verifyUnknownProfileMetadata(MDNode *MD) { Check(MD->getNumOperands() == 2, @@ -2677,6 +2736,14 @@ void Verifier::visitConstantPtrAuth(const ConstantPtrAuth *CPA) { Check(CPA->getDiscriminator()->getBitWidth() == 64, "signed ptrauth constant discriminator must be i64 constant integer"); + + Check(CPA->getDeactivationSymbol()->getType()->isPointerTy(), + "signed ptrauth constant deactivation symbol must be a pointer"); + + Check(isa<GlobalValue>(CPA->getDeactivationSymbol()) || + CPA->getDeactivationSymbol()->isNullValue(), + "signed ptrauth constant deactivation symbol must be a global value " + "or null"); } bool Verifier::verifyAttributeCount(AttributeList Attrs, unsigned Params) { @@ -3379,11 +3446,34 @@ void Verifier::visitIndirectBrInst(IndirectBrInst &BI) { } void Verifier::visitCallBrInst(CallBrInst &CBI) { - Check(CBI.isInlineAsm(), "Callbr is currently only used for asm-goto!", &CBI); - const InlineAsm *IA = cast<InlineAsm>(CBI.getCalledOperand()); - Check(!IA->canThrow(), "Unwinding from Callbr is not allowed"); + if (!CBI.isInlineAsm()) { + Check(CBI.getCalledFunction(), + "Callbr: indirect function / invalid signature"); + Check(!CBI.hasOperandBundles(), + "Callbr for intrinsics currently doesn't support operand bundles"); + + switch (CBI.getIntrinsicID()) { + case Intrinsic::amdgcn_kill: { + Check(CBI.getNumIndirectDests() == 1, + "Callbr amdgcn_kill only supports one indirect dest"); + bool Unreachable = isa<UnreachableInst>(CBI.getIndirectDest(0)->begin()); + CallInst *Call = dyn_cast<CallInst>(CBI.getIndirectDest(0)->begin()); + Check(Unreachable || (Call && Call->getIntrinsicID() == + Intrinsic::amdgcn_unreachable), + "Callbr amdgcn_kill indirect dest needs to be unreachable"); + break; + } + default: + CheckFailed( + "Callbr currently only supports asm-goto and selected intrinsics"); + } + visitIntrinsicCall(CBI.getIntrinsicID(), CBI); + } else { + const InlineAsm *IA = cast<InlineAsm>(CBI.getCalledOperand()); + Check(!IA->canThrow(), "Unwinding from Callbr is not allowed"); - verifyInlineAsmCall(CBI); + verifyInlineAsmCall(CBI); + } visitTerminator(CBI); } @@ -5479,7 +5569,7 @@ void Verifier::visitInstruction(Instruction &I) { (CBI && &CBI->getCalledOperandUse() == &I.getOperandUse(i)) || IsAttachedCallOperand(F, CBI, i)), "Cannot take the address of an intrinsic!", &I); - Check(!F->isIntrinsic() || isa<CallInst>(I) || + Check(!F->isIntrinsic() || isa<CallInst>(I) || isa<CallBrInst>(I) || F->getIntrinsicID() == Intrinsic::donothing || F->getIntrinsicID() == Intrinsic::seh_try_begin || F->getIntrinsicID() == Intrinsic::seh_try_end || @@ -6017,6 +6107,12 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { Check(cast<ConstantInt>(Call.getArgOperand(3))->getZExtValue() < 2, "cache type argument to llvm.prefetch must be 0-1", Call); break; + case Intrinsic::reloc_none: { + Check(isa<MDString>( + cast<MetadataAsValue>(Call.getArgOperand(0))->getMetadata()), + "llvm.reloc.none argument must be a metadata string", &Call); + break; + } case Intrinsic::stackprotector: Check(isa<AllocaInst>(Call.getArgOperand(1)->stripPointerCasts()), "llvm.stackprotector parameter #2 must resolve to an alloca.", Call); @@ -6474,7 +6570,8 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { VectorType *VecTy = cast<VectorType>(Call.getType()); int64_t Idx = cast<ConstantInt>(Call.getArgOperand(2))->getSExtValue(); int64_t KnownMinNumElements = VecTy->getElementCount().getKnownMinValue(); - if (Call.getParent() && Call.getParent()->getParent()) { + if (VecTy->isScalableTy() && Call.getParent() && + Call.getParent()->getParent()) { AttributeList Attrs = Call.getParent()->getParent()->getAttributes(); if (Attrs.hasFnAttr(Attribute::VScaleRange)) KnownMinNumElements *= Attrs.getFnAttrs().getVScaleRangeMin(); @@ -6581,6 +6678,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { } break; } + case Intrinsic::vector_partial_reduce_fadd: case Intrinsic::vector_partial_reduce_add: { VectorType *AccTy = cast<VectorType>(Call.getArgOperand(0)->getType()); VectorType *VecTy = cast<VectorType>(Call.getArgOperand(1)->getType()); @@ -7024,6 +7122,7 @@ void Verifier::visit(DbgVariableRecord &DVR) { CheckDI(DVR.getType() == DbgVariableRecord::LocationType::Value || DVR.getType() == DbgVariableRecord::LocationType::Declare || + DVR.getType() == DbgVariableRecord::LocationType::DeclareValue || DVR.getType() == DbgVariableRecord::LocationType::Assign, "invalid #dbg record type", &DVR, DVR.getType(), BB, F); |