diff options
Diffstat (limited to 'llvm/include')
119 files changed, 1695 insertions, 797 deletions
diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h index ff8bdb8..fb91690 100644 --- a/llvm/include/llvm/ADT/ArrayRef.h +++ b/llvm/include/llvm/ADT/ArrayRef.h @@ -317,10 +317,6 @@ namespace llvm { /// Construct an empty MutableArrayRef. /*implicit*/ MutableArrayRef() = default; - /// Construct an empty MutableArrayRef from std::nullopt. - /*implicit*/ LLVM_DEPRECATED("Use {} or MutableArrayRef<T>() instead", "{}") - MutableArrayRef(std::nullopt_t) : ArrayRef<T>() {} - /// Construct a MutableArrayRef from a single element. /*implicit*/ MutableArrayRef(T &OneElt) : ArrayRef<T>(OneElt) {} diff --git a/llvm/include/llvm/ADT/DenseMapInfo.h b/llvm/include/llvm/ADT/DenseMapInfo.h index b850223..9d8fd89 100644 --- a/llvm/include/llvm/ADT/DenseMapInfo.h +++ b/llvm/include/llvm/ADT/DenseMapInfo.h @@ -51,10 +51,10 @@ inline unsigned combineHashValue(unsigned a, unsigned b) { /// just be `void`. template<typename T, typename Enable = void> struct DenseMapInfo { - //static inline T getEmptyKey(); - //static inline T getTombstoneKey(); - //static unsigned getHashValue(const T &Val); - //static bool isEqual(const T &LHS, const T &RHS); + // static constexpr T getEmptyKey(); + // static constexpr T getTombstoneKey(); + // static unsigned getHashValue(const T &Val); + // static bool isEqual(const T &LHS, const T &RHS); }; // Provide DenseMapInfo for all pointers. Come up with sentinel pointer values @@ -70,13 +70,13 @@ struct DenseMapInfo<T*> { // "Log2MaxAlign bits of alignment"); static constexpr uintptr_t Log2MaxAlign = 12; - static inline T* getEmptyKey() { + static constexpr T *getEmptyKey() { uintptr_t Val = static_cast<uintptr_t>(-1); Val <<= Log2MaxAlign; return reinterpret_cast<T*>(Val); } - static inline T* getTombstoneKey() { + static constexpr T *getTombstoneKey() { uintptr_t Val = static_cast<uintptr_t>(-2); Val <<= Log2MaxAlign; return reinterpret_cast<T*>(Val); @@ -92,8 +92,8 @@ struct DenseMapInfo<T*> { // Provide DenseMapInfo for chars. template<> struct DenseMapInfo<char> { - static inline char getEmptyKey() { return ~0; } - static inline char getTombstoneKey() { return ~0 - 1; } + static constexpr char getEmptyKey() { return ~0; } + static constexpr char getTombstoneKey() { return ~0 - 1; } static unsigned getHashValue(const char& Val) { return Val * 37U; } static bool isEqual(const char &LHS, const char &RHS) { @@ -103,8 +103,8 @@ template<> struct DenseMapInfo<char> { // Provide DenseMapInfo for unsigned chars. template <> struct DenseMapInfo<unsigned char> { - static inline unsigned char getEmptyKey() { return ~0; } - static inline unsigned char getTombstoneKey() { return ~0 - 1; } + static constexpr unsigned char getEmptyKey() { return ~0; } + static constexpr unsigned char getTombstoneKey() { return ~0 - 1; } static unsigned getHashValue(const unsigned char &Val) { return Val * 37U; } static bool isEqual(const unsigned char &LHS, const unsigned char &RHS) { @@ -114,8 +114,8 @@ template <> struct DenseMapInfo<unsigned char> { // Provide DenseMapInfo for unsigned shorts. template <> struct DenseMapInfo<unsigned short> { - static inline unsigned short getEmptyKey() { return 0xFFFF; } - static inline unsigned short getTombstoneKey() { return 0xFFFF - 1; } + static constexpr unsigned short getEmptyKey() { return 0xFFFF; } + static constexpr unsigned short getTombstoneKey() { return 0xFFFF - 1; } static unsigned getHashValue(const unsigned short &Val) { return Val * 37U; } static bool isEqual(const unsigned short &LHS, const unsigned short &RHS) { @@ -125,8 +125,8 @@ template <> struct DenseMapInfo<unsigned short> { // Provide DenseMapInfo for unsigned ints. template<> struct DenseMapInfo<unsigned> { - static inline unsigned getEmptyKey() { return ~0U; } - static inline unsigned getTombstoneKey() { return ~0U - 1; } + static constexpr unsigned getEmptyKey() { return ~0U; } + static constexpr unsigned getTombstoneKey() { return ~0U - 1; } static unsigned getHashValue(const unsigned& Val) { return Val * 37U; } static bool isEqual(const unsigned& LHS, const unsigned& RHS) { @@ -136,8 +136,8 @@ template<> struct DenseMapInfo<unsigned> { // Provide DenseMapInfo for unsigned longs. template<> struct DenseMapInfo<unsigned long> { - static inline unsigned long getEmptyKey() { return ~0UL; } - static inline unsigned long getTombstoneKey() { return ~0UL - 1L; } + static constexpr unsigned long getEmptyKey() { return ~0UL; } + static constexpr unsigned long getTombstoneKey() { return ~0UL - 1L; } static unsigned getHashValue(const unsigned long& Val) { if constexpr (sizeof(Val) == 4) @@ -153,8 +153,8 @@ template<> struct DenseMapInfo<unsigned long> { // Provide DenseMapInfo for unsigned long longs. template<> struct DenseMapInfo<unsigned long long> { - static inline unsigned long long getEmptyKey() { return ~0ULL; } - static inline unsigned long long getTombstoneKey() { return ~0ULL - 1ULL; } + static constexpr unsigned long long getEmptyKey() { return ~0ULL; } + static constexpr unsigned long long getTombstoneKey() { return ~0ULL - 1ULL; } static unsigned getHashValue(const unsigned long long& Val) { return densemap::detail::mix(Val); @@ -168,16 +168,16 @@ template<> struct DenseMapInfo<unsigned long long> { // Provide DenseMapInfo for shorts. template <> struct DenseMapInfo<short> { - static inline short getEmptyKey() { return 0x7FFF; } - static inline short getTombstoneKey() { return -0x7FFF - 1; } + static constexpr short getEmptyKey() { return 0x7FFF; } + static constexpr short getTombstoneKey() { return -0x7FFF - 1; } static unsigned getHashValue(const short &Val) { return Val * 37U; } static bool isEqual(const short &LHS, const short &RHS) { return LHS == RHS; } }; // Provide DenseMapInfo for ints. template<> struct DenseMapInfo<int> { - static inline int getEmptyKey() { return 0x7fffffff; } - static inline int getTombstoneKey() { return -0x7fffffff - 1; } + static constexpr int getEmptyKey() { return 0x7fffffff; } + static constexpr int getTombstoneKey() { return -0x7fffffff - 1; } static unsigned getHashValue(const int& Val) { return (unsigned)(Val * 37U); } static bool isEqual(const int& LHS, const int& RHS) { @@ -187,11 +187,11 @@ template<> struct DenseMapInfo<int> { // Provide DenseMapInfo for longs. template<> struct DenseMapInfo<long> { - static inline long getEmptyKey() { + static constexpr long getEmptyKey() { return (1UL << (sizeof(long) * 8 - 1)) - 1UL; } - static inline long getTombstoneKey() { return getEmptyKey() - 1L; } + static constexpr long getTombstoneKey() { return getEmptyKey() - 1L; } static unsigned getHashValue(const long& Val) { return (unsigned)(Val * 37UL); @@ -204,8 +204,10 @@ template<> struct DenseMapInfo<long> { // Provide DenseMapInfo for long longs. template<> struct DenseMapInfo<long long> { - static inline long long getEmptyKey() { return 0x7fffffffffffffffLL; } - static inline long long getTombstoneKey() { return -0x7fffffffffffffffLL-1; } + static constexpr long long getEmptyKey() { return 0x7fffffffffffffffLL; } + static constexpr long long getTombstoneKey() { + return -0x7fffffffffffffffLL - 1; + } static unsigned getHashValue(const long long& Val) { return (unsigned)(Val * 37ULL); @@ -224,12 +226,12 @@ struct DenseMapInfo<std::pair<T, U>> { using FirstInfo = DenseMapInfo<T>; using SecondInfo = DenseMapInfo<U>; - static inline Pair getEmptyKey() { + static constexpr Pair getEmptyKey() { return std::make_pair(FirstInfo::getEmptyKey(), SecondInfo::getEmptyKey()); } - static inline Pair getTombstoneKey() { + static constexpr Pair getTombstoneKey() { return std::make_pair(FirstInfo::getTombstoneKey(), SecondInfo::getTombstoneKey()); } @@ -257,11 +259,11 @@ struct DenseMapInfo<std::pair<T, U>> { template <typename... Ts> struct DenseMapInfo<std::tuple<Ts...>> { using Tuple = std::tuple<Ts...>; - static inline Tuple getEmptyKey() { + static constexpr Tuple getEmptyKey() { return Tuple(DenseMapInfo<Ts>::getEmptyKey()...); } - static inline Tuple getTombstoneKey() { + static constexpr Tuple getTombstoneKey() { return Tuple(DenseMapInfo<Ts>::getTombstoneKey()...); } @@ -309,10 +311,22 @@ struct DenseMapInfo<Enum, std::enable_if_t<std::is_enum_v<Enum>>> { using UnderlyingType = std::underlying_type_t<Enum>; using Info = DenseMapInfo<UnderlyingType>; - static Enum getEmptyKey() { return static_cast<Enum>(Info::getEmptyKey()); } + // If an enum does not have a "fixed" underlying type, it may be UB to cast + // some values of the underlying type to the enum. We use an "extra" constexpr + // local to ensure that such UB would trigger "static assertion expression is + // not an integral constant expression", rather than runtime UB. + // + // If you hit this error, you can fix by switching to `enum class`, or adding + // an explicit underlying type (e.g. `enum X : int`) to the enum's definition. + + static constexpr Enum getEmptyKey() { + constexpr Enum V = static_cast<Enum>(Info::getEmptyKey()); + return V; + } - static Enum getTombstoneKey() { - return static_cast<Enum>(Info::getTombstoneKey()); + static constexpr Enum getTombstoneKey() { + constexpr Enum V = static_cast<Enum>(Info::getTombstoneKey()); + return V; } static unsigned getHashValue(const Enum &Val) { @@ -326,9 +340,11 @@ template <typename T> struct DenseMapInfo<std::optional<T>> { using Optional = std::optional<T>; using Info = DenseMapInfo<T>; - static inline Optional getEmptyKey() { return {Info::getEmptyKey()}; } + static constexpr Optional getEmptyKey() { return {Info::getEmptyKey()}; } - static inline Optional getTombstoneKey() { return {Info::getTombstoneKey()}; } + static constexpr Optional getTombstoneKey() { + return {Info::getTombstoneKey()}; + } static unsigned getHashValue(const Optional &OptionalVal) { return detail::combineHashValue( diff --git a/llvm/include/llvm/Analysis/DXILResource.h b/llvm/include/llvm/Analysis/DXILResource.h index 9e2dc1a..956dcbc 100644 --- a/llvm/include/llvm/Analysis/DXILResource.h +++ b/llvm/include/llvm/Analysis/DXILResource.h @@ -359,6 +359,8 @@ public: std::tie(RHS.RecordID, RHS.Space, RHS.LowerBound, RHS.Size); } bool overlapsWith(const ResourceBinding &RHS) const { + if (Size == UINT32_MAX) + return LowerBound < RHS.LowerBound; return Space == RHS.Space && LowerBound + Size - 1 >= RHS.LowerBound; } }; diff --git a/llvm/include/llvm/Analysis/IR2Vec.h b/llvm/include/llvm/Analysis/IR2Vec.h index d87457c..17f4112 100644 --- a/llvm/include/llvm/Analysis/IR2Vec.h +++ b/llvm/include/llvm/Analysis/IR2Vec.h @@ -153,6 +153,7 @@ class Vocabulary { static_cast<unsigned>(OperandKind::MaxOperandKind), "OperandKindNames array size must match MaxOperandKind"); +public: /// Vocabulary layout constants #define LAST_OTHER_INST(NUM) static constexpr unsigned MaxOpcodes = NUM; #include "llvm/IR/Instruction.def" @@ -162,39 +163,38 @@ class Vocabulary { static constexpr unsigned MaxOperandKinds = static_cast<unsigned>(OperandKind::MaxOperandKind); -public: Vocabulary() = default; - Vocabulary(VocabVector &&Vocab); + LLVM_ABI Vocabulary(VocabVector &&Vocab); - bool isValid() const; - unsigned getDimension() const; - size_t size() const; + LLVM_ABI bool isValid() const; + LLVM_ABI unsigned getDimension() const; + LLVM_ABI size_t size() const; static size_t expectedSize() { return MaxOpcodes + MaxTypeIDs + MaxOperandKinds; } /// Helper function to get vocabulary key for a given Opcode - static StringRef getVocabKeyForOpcode(unsigned Opcode); + LLVM_ABI static StringRef getVocabKeyForOpcode(unsigned Opcode); /// Helper function to get vocabulary key for a given TypeID - static StringRef getVocabKeyForTypeID(Type::TypeID TypeID); + LLVM_ABI static StringRef getVocabKeyForTypeID(Type::TypeID TypeID); /// Helper function to get vocabulary key for a given OperandKind - static StringRef getVocabKeyForOperandKind(OperandKind Kind); + LLVM_ABI static StringRef getVocabKeyForOperandKind(OperandKind Kind); /// Helper function to classify an operand into OperandKind - static OperandKind getOperandKind(const Value *Op); + LLVM_ABI static OperandKind getOperandKind(const Value *Op); /// Helpers to return the IDs of a given Opcode, TypeID, or OperandKind - static unsigned getNumericID(unsigned Opcode); - static unsigned getNumericID(Type::TypeID TypeID); - static unsigned getNumericID(const Value *Op); + LLVM_ABI static unsigned getNumericID(unsigned Opcode); + LLVM_ABI static unsigned getNumericID(Type::TypeID TypeID); + LLVM_ABI static unsigned getNumericID(const Value *Op); /// Accessors to get the embedding for a given entity. - const ir2vec::Embedding &operator[](unsigned Opcode) const; - const ir2vec::Embedding &operator[](Type::TypeID TypeId) const; - const ir2vec::Embedding &operator[](const Value *Arg) const; + LLVM_ABI const ir2vec::Embedding &operator[](unsigned Opcode) const; + LLVM_ABI const ir2vec::Embedding &operator[](Type::TypeID TypeId) const; + LLVM_ABI const ir2vec::Embedding &operator[](const Value *Arg) const; /// Const Iterator type aliases using const_iterator = VocabVector::const_iterator; @@ -221,13 +221,13 @@ public: /// Returns the string key for a given index position in the vocabulary. /// This is useful for debugging or printing the vocabulary. Do not use this /// for embedding generation as string based lookups are inefficient. - static StringRef getStringKey(unsigned Pos); + LLVM_ABI static StringRef getStringKey(unsigned Pos); /// Create a dummy vocabulary for testing purposes. - static VocabVector createDummyVocabForTest(unsigned Dim = 1); + LLVM_ABI static VocabVector createDummyVocabForTest(unsigned Dim = 1); - bool invalidate(Module &M, const PreservedAnalyses &PA, - ModuleAnalysisManager::Invalidator &Inv) const; + LLVM_ABI bool invalidate(Module &M, const PreservedAnalyses &PA, + ModuleAnalysisManager::Invalidator &Inv) const; }; /// Embedder provides the interface to generate embeddings (vector diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h index 73bfe1a..af6e534 100644 --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -236,8 +236,8 @@ public: /// In same cases when the dependency check fails we can still /// vectorize the loop with a dynamic array access check. - bool shouldRetryWithRuntimeCheck() const { - return FoundNonConstantDistanceDependence && + bool shouldRetryWithRuntimeChecks() const { + return ShouldRetryWithRuntimeChecks && Status == VectorizationSafetyStatus::PossiblySafeWithRtChecks; } @@ -327,9 +327,9 @@ private: uint64_t MaxStoreLoadForwardSafeDistanceInBits = std::numeric_limits<uint64_t>::max(); - /// If we see a non-constant dependence distance we can still try to - /// vectorize this loop with runtime checks. - bool FoundNonConstantDistanceDependence = false; + /// Whether we should try to vectorize the loop with runtime checks, if the + /// dependencies are not safe. + bool ShouldRetryWithRuntimeChecks = false; /// Result of the dependence checks, indicating whether the checked /// dependences are safe for vectorization, require RT checks or are known to diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h index 09e3945..bff7707 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h @@ -160,6 +160,12 @@ m_scev_ZExt(const Op0_t &Op0) { return m_scev_Unary<SCEVZeroExtendExpr>(Op0); } +template <typename Op0_t> +inline SCEVUnaryExpr_match<SCEVPtrToIntExpr, Op0_t> +m_scev_PtrToInt(const Op0_t &Op0) { + return SCEVUnaryExpr_match<SCEVPtrToIntExpr, Op0_t>(Op0); +} + /// Match a binary SCEV. template <typename SCEVTy, typename Op0_t, typename Op1_t> struct SCEVBinaryExpr_match { diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 98b793a..7928835 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -1930,7 +1930,7 @@ public: /// Returns a bitmask constructed from the target-features or fmv-features /// metadata of a function. - LLVM_ABI uint64_t getFeatureMask(const Function &F) const; + LLVM_ABI APInt getFeatureMask(const Function &F) const; /// Returns true if this is an instance of a function with multiple versions. LLVM_ABI bool isMultiversionedFunction(const Function &F) const; diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index ddc8a5e..2ea87b3 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -1126,7 +1126,9 @@ public: virtual bool hasArmWideBranch(bool) const { return false; } - virtual uint64_t getFeatureMask(const Function &F) const { return 0; } + virtual APInt getFeatureMask(const Function &F) const { + return APInt::getZero(32); + } virtual bool isMultiversionedFunction(const Function &F) const { return false; diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h index 9a2773c..b55c4e0 100644 --- a/llvm/include/llvm/Analysis/VectorUtils.h +++ b/llvm/include/llvm/Analysis/VectorUtils.h @@ -177,12 +177,6 @@ LLVM_ABI bool isVectorIntrinsicWithStructReturnOverloadAtField( LLVM_ABI Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI); -/// Returns the corresponding llvm.vector.interleaveN intrinsic for factor N. -LLVM_ABI Intrinsic::ID getInterleaveIntrinsicID(unsigned Factor); - -/// Returns the corresponding llvm.vector.deinterleaveN intrinsic for factor N. -LLVM_ABI Intrinsic::ID getDeinterleaveIntrinsicID(unsigned Factor); - /// Returns the corresponding factor of llvm.vector.interleaveN intrinsics. LLVM_ABI unsigned getInterleaveIntrinsicFactor(Intrinsic::ID ID); diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h index c7e4bdf..a2311d2 100644 --- a/llvm/include/llvm/AsmParser/LLToken.h +++ b/llvm/include/llvm/AsmParser/LLToken.h @@ -181,6 +181,7 @@ enum Kind { kw_amdgpu_cs_chain_preserve, kw_amdgpu_kernel, kw_amdgpu_gfx, + kw_amdgpu_gfx_whole_wave, kw_tailcc, kw_m68k_rtdcc, kw_graalcc, diff --git a/llvm/include/llvm/BinaryFormat/COFF.h b/llvm/include/llvm/BinaryFormat/COFF.h index f3b5d5e..64fe216 100644 --- a/llvm/include/llvm/BinaryFormat/COFF.h +++ b/llvm/include/llvm/BinaryFormat/COFF.h @@ -694,7 +694,24 @@ enum DLLCharacteristics : unsigned { enum ExtendedDLLCharacteristics : unsigned { /// Image is CET compatible - IMAGE_DLL_CHARACTERISTICS_EX_CET_COMPAT = 0x0001 + IMAGE_DLL_CHARACTERISTICS_EX_CET_COMPAT = 0x0001, + /// Image is CET compatible in strict mode + IMAGE_DLL_CHARACTERISTICS_EX_CET_COMPAT_STRICT_MODE = 0x0002, + /// Image is CET compatible in such a way that context IP validation is + /// relaxed + IMAGE_DLL_CHARACTERISTICS_EX_CET_SET_CONTEXT_IP_VALIDATION_RELAXED_MODE = + 0x0004, + /// Image is CET compatible in such a way that the use of + /// dynamic APIs is restricted to processes only + IMAGE_DLL_CHARACTERISTICS_EX_CET_DYNAMIC_APIS_ALLOW_IN_PROC_ONLY = 0x0008, + /// Reserved for future use. Not used by MSVC link.exe + IMAGE_DLL_CHARACTERISTICS_EX_CET_RESERVED_1 = 0x0010, + /// Reserved for future use. Not used by MSVC link.exe + IMAGE_DLL_CHARACTERISTICS_EX_CET_RESERVED_2 = 0x0020, + /// Image is CFI compatible. + IMAGE_DLL_CHARACTERISTICS_EX_FORWARD_CFI_COMPAT = 0x0040, + /// Image is hotpatch compatible. + IMAGE_DLL_CHARACTERISTICS_EX_HOTPATCH_COMPATIBLE = 0x0080, }; enum DebugType : unsigned { diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index e4f82ad..ad35d7f 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -362,6 +362,7 @@ enum { ELFOSABI_FENIXOS = 16, // FenixOS ELFOSABI_CLOUDABI = 17, // Nuxi CloudABI ELFOSABI_CUDA = 51, // NVIDIA CUDA architecture. + ELFOSABI_CUDA_V2 = 41, // NVIDIA CUDA architecture. ELFOSABI_FIRST_ARCH = 64, // First architecture-specific OS ABI ELFOSABI_AMDGPU_HSA = 64, // AMD HSA runtime ELFOSABI_AMDGPU_PAL = 65, // AMD PAL runtime @@ -385,6 +386,12 @@ enum { ELFABIVERSION_AMDGPU_HSA_V6 = 4, }; +// CUDA OS ABI Version identification. +enum { + ELFABIVERSION_CUDA_V1 = 7, + ELFABIVERSION_CUDA_V2 = 8, +}; + #define ELF_RELOC(name, value) name = value, // X86_64 relocations. @@ -921,7 +928,7 @@ enum { // NVPTX specific e_flags. enum : unsigned { - // Processor selection mask for EF_CUDA_SM* values. + // Processor selection mask for EF_CUDA_SM* values prior to blackwell. EF_CUDA_SM = 0xff, // SM based processor values. @@ -954,12 +961,22 @@ enum : unsigned { // The target is using 64-bit addressing. EF_CUDA_64BIT_ADDRESS = 0x400, // Set when using the sm_90a processor. - EF_CUDA_ACCELERATORS = 0x800, + EF_CUDA_ACCELERATORS_V1 = 0x800, // Undocumented software feature. EF_CUDA_SW_FLAG_V2 = 0x1000, // Virtual processor selection mask for EF_CUDA_VIRTUAL_SM* values. EF_CUDA_VIRTUAL_SM = 0xff0000, + + // Processor selection mask for EF_CUDA_SM* values following blackwell. + EF_CUDA_SM_MASK = 0xff00, + + // SM based processor values. + EF_CUDA_SM100 = 0x6400, + EF_CUDA_SM120 = 0x7800, + + // Set when using an accelerator variant like sm_100a. + EF_CUDA_ACCELERATORS = 0x8, }; // ELF Relocation types for BPF diff --git a/llvm/include/llvm/BinaryFormat/SFrame.h b/llvm/include/llvm/BinaryFormat/SFrame.h index 16d3b16..0c6c4d1 100644 --- a/llvm/include/llvm/BinaryFormat/SFrame.h +++ b/llvm/include/llvm/BinaryFormat/SFrame.h @@ -15,60 +15,62 @@ #ifndef LLVM_BINARYFORMAT_SFRAME_H #define LLVM_BINARYFORMAT_SFRAME_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitmaskEnum.h" +#include "llvm/Support/Compiler.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/Endian.h" -namespace llvm::sframe { +namespace llvm { + +template <typename T> struct EnumEntry; + +namespace sframe { LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); constexpr uint16_t Magic = 0xdee2; enum class Version : uint8_t { - V1 = 1, - V2 = 2, +#define HANDLE_SFRAME_VERSION(CODE, NAME) NAME = CODE, +#include "llvm/BinaryFormat/SFrameConstants.def" }; enum class Flags : uint8_t { - FDESorted = 0x01, - FramePointer = 0x02, - FDEFuncStartPCRel = 0x04, +#define HANDLE_SFRAME_FLAG(CODE, NAME) NAME = CODE, +#include "llvm/BinaryFormat/SFrameConstants.def" V2AllFlags = FDESorted | FramePointer | FDEFuncStartPCRel, LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/0xff), }; enum class ABI : uint8_t { - AArch64EndianBig = 1, - AArch64EndianLittle = 2, - AMD64EndianLittle = 3, +#define HANDLE_SFRAME_ABI(CODE, NAME) NAME = CODE, +#include "llvm/BinaryFormat/SFrameConstants.def" }; /// SFrame FRE Types. Bits 0-3 of FuncDescEntry.Info. enum class FREType : uint8_t { - Addr1 = 0, - Addr2 = 1, - Addr4 = 2, +#define HANDLE_SFRAME_FRE_TYPE(CODE, NAME) NAME = CODE, +#include "llvm/BinaryFormat/SFrameConstants.def" }; /// SFrame FDE Types. Bit 4 of FuncDescEntry.Info. enum class FDEType : uint8_t { - PCInc = 0, - PCMask = 1, +#define HANDLE_SFRAME_FDE_TYPE(CODE, NAME) NAME = CODE, +#include "llvm/BinaryFormat/SFrameConstants.def" }; /// Speficies key used for signing return addresses. Bit 5 of /// FuncDescEntry.Info. enum class AArch64PAuthKey : uint8_t { - A = 0, - B = 1, +#define HANDLE_SFRAME_AARCH64_PAUTH_KEY(CODE, NAME) NAME = CODE, +#include "llvm/BinaryFormat/SFrameConstants.def" }; -/// Size of stack offsets. Bits 5-6 of FREInfo.Info. +/// Size of stack offsets. Bits 6-7 of FREInfo.Info. enum class FREOffset : uint8_t { - B1 = 0, - B2 = 1, - B4 = 2, +#define HANDLE_SFRAME_FRE_OFFSET(CODE, NAME) NAME = CODE, +#include "llvm/BinaryFormat/SFrameConstants.def" }; /// Stack frame base register. Bit 0 of FREInfo.Info. @@ -160,6 +162,15 @@ template <endianness E> using FrameRowEntryAddr1 = FrameRowEntry<uint8_t, E>; template <endianness E> using FrameRowEntryAddr2 = FrameRowEntry<uint16_t, E>; template <endianness E> using FrameRowEntryAddr4 = FrameRowEntry<uint32_t, E>; -} // namespace llvm::sframe +LLVM_ABI ArrayRef<EnumEntry<Version>> getVersions(); +LLVM_ABI ArrayRef<EnumEntry<Flags>> getFlags(); +LLVM_ABI ArrayRef<EnumEntry<ABI>> getABIs(); +LLVM_ABI ArrayRef<EnumEntry<FREType>> getFRETypes(); +LLVM_ABI ArrayRef<EnumEntry<FDEType>> getFDETypes(); +LLVM_ABI ArrayRef<EnumEntry<AArch64PAuthKey>> getAArch64PAuthKeys(); +LLVM_ABI ArrayRef<EnumEntry<FREOffset>> getFREOffsets(); + +} // namespace sframe +} // namespace llvm #endif // LLVM_BINARYFORMAT_SFRAME_H diff --git a/llvm/include/llvm/BinaryFormat/SFrameConstants.def b/llvm/include/llvm/BinaryFormat/SFrameConstants.def new file mode 100644 index 0000000..fddd440 --- /dev/null +++ b/llvm/include/llvm/BinaryFormat/SFrameConstants.def @@ -0,0 +1,76 @@ +//===- SFrameConstants.def --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if !(defined(HANDLE_SFRAME_VERSION) || defined(HANDLE_SFRAME_FLAG) || \ + defined(HANDLE_SFRAME_ABI) || defined(HANDLE_SFRAME_FRE_TYPE) || \ + defined(HANDLE_SFRAME_FDE_TYPE) || \ + defined(HANDLE_SFRAME_AARCH64_PAUTH_KEY) || \ + defined(HANDLE_SFRAME_FRE_OFFSET)) +#error "Missing HANDLE_SFRAME definition" +#endif + +#ifndef HANDLE_SFRAME_VERSION +#define HANDLE_SFRAME_VERSION(CODE, NAME) +#endif + +#ifndef HANDLE_SFRAME_FLAG +#define HANDLE_SFRAME_FLAG(CODE, NAME) +#endif + +#ifndef HANDLE_SFRAME_ABI +#define HANDLE_SFRAME_ABI(CODE, NAME) +#endif + +#ifndef HANDLE_SFRAME_FRE_TYPE +#define HANDLE_SFRAME_FRE_TYPE(CODE, NAME) +#endif + +#ifndef HANDLE_SFRAME_FDE_TYPE +#define HANDLE_SFRAME_FDE_TYPE(CODE, NAME) +#endif + +#ifndef HANDLE_SFRAME_AARCH64_PAUTH_KEY +#define HANDLE_SFRAME_AARCH64_PAUTH_KEY(CODE, NAME) +#endif + +#ifndef HANDLE_SFRAME_FRE_OFFSET +#define HANDLE_SFRAME_FRE_OFFSET(CODE, NAME) +#endif + +HANDLE_SFRAME_VERSION(0x01, V1) +HANDLE_SFRAME_VERSION(0x02, V2) + +HANDLE_SFRAME_FLAG(0x01, FDESorted) +HANDLE_SFRAME_FLAG(0x02, FramePointer) +HANDLE_SFRAME_FLAG(0x04, FDEFuncStartPCRel) + +HANDLE_SFRAME_ABI(0x01, AArch64EndianBig) +HANDLE_SFRAME_ABI(0x02, AArch64EndianLittle) +HANDLE_SFRAME_ABI(0x03, AMD64EndianLittle) + +HANDLE_SFRAME_FRE_TYPE(0x00, Addr1) +HANDLE_SFRAME_FRE_TYPE(0x01, Addr2) +HANDLE_SFRAME_FRE_TYPE(0x02, Addr4) + +HANDLE_SFRAME_FDE_TYPE(0, PCInc) +HANDLE_SFRAME_FDE_TYPE(1, PCMask) + +HANDLE_SFRAME_AARCH64_PAUTH_KEY(0, A) +HANDLE_SFRAME_AARCH64_PAUTH_KEY(1, B) + +HANDLE_SFRAME_FRE_OFFSET(0, B1) +HANDLE_SFRAME_FRE_OFFSET(1, B2) +HANDLE_SFRAME_FRE_OFFSET(2, B4) + +#undef HANDLE_SFRAME_VERSION +#undef HANDLE_SFRAME_FLAG +#undef HANDLE_SFRAME_ABI +#undef HANDLE_SFRAME_FRE_TYPE +#undef HANDLE_SFRAME_FDE_TYPE +#undef HANDLE_SFRAME_AARCH64_PAUTH_KEY +#undef HANDLE_SFRAME_FRE_OFFSET diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 1d7c414..1fcedcd 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -1985,11 +1985,6 @@ public: cast<VectorType>(Args[0]->getType()), {}, CostKind, Index, cast<VectorType>(Args[1]->getType())); } - case Intrinsic::vector_reverse: { - return thisT()->getShuffleCost(TTI::SK_Reverse, cast<VectorType>(RetTy), - cast<VectorType>(Args[0]->getType()), {}, - CostKind, 0, cast<VectorType>(RetTy)); - } case Intrinsic::vector_splice: { unsigned Index = cast<ConstantInt>(Args[2])->getZExtValue(); return thisT()->getShuffleCost(TTI::SK_Splice, cast<VectorType>(RetTy), @@ -2458,6 +2453,10 @@ public: thisT()->getArithmeticInstrCost(BinaryOperator::And, RetTy, CostKind); return Cost; } + case Intrinsic::vector_reverse: + return thisT()->getShuffleCost(TTI::SK_Reverse, cast<VectorType>(RetTy), + cast<VectorType>(ICA.getArgTypes()[0]), {}, + CostKind, 0, cast<VectorType>(RetTy)); case Intrinsic::get_active_lane_mask: { Type *ArgTy = ICA.getArgTypes()[0]; EVT ResVT = getTLI()->getValueType(DL, RetTy, true); diff --git a/llvm/include/llvm/CodeGen/CommandFlags.h b/llvm/include/llvm/CodeGen/CommandFlags.h index aefdb53..d500e94 100644 --- a/llvm/include/llvm/CodeGen/CommandFlags.h +++ b/llvm/include/llvm/CodeGen/CommandFlags.h @@ -133,6 +133,8 @@ LLVM_ABI bool getEnableStackSizeSection(); LLVM_ABI bool getEnableAddrsig(); +LLVM_ABI bool getEnableCallGraphSection(); + LLVM_ABI bool getEmitCallSiteInfo(); LLVM_ABI bool getEnableMachineFunctionSplitter(); diff --git a/llvm/include/llvm/CodeGen/GCMetadata.h b/llvm/include/llvm/CodeGen/GCMetadata.h index 33f5301..5b9ee28 100644 --- a/llvm/include/llvm/CodeGen/GCMetadata.h +++ b/llvm/include/llvm/CodeGen/GCMetadata.h @@ -101,12 +101,12 @@ private: // are live per safe point (1.5% on 64-bit hosts). public: - GCFunctionInfo(const Function &F, GCStrategy &S); - ~GCFunctionInfo(); + LLVM_ABI GCFunctionInfo(const Function &F, GCStrategy &S); + LLVM_ABI ~GCFunctionInfo(); /// Handle invalidation explicitly. - bool invalidate(Function &F, const PreservedAnalyses &PA, - FunctionAnalysisManager::Invalidator &Inv); + LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA, + FunctionAnalysisManager::Invalidator &Inv); /// getFunction - Return the function to which this metadata applies. const Function &getFunction() const { return F; } @@ -163,8 +163,8 @@ public: GCStrategyMap(GCStrategyMap &&) = default; /// Handle invalidation explicitly. - bool invalidate(Module &M, const PreservedAnalyses &PA, - ModuleAnalysisManager::Invalidator &Inv); + LLVM_ABI bool invalidate(Module &M, const PreservedAnalyses &PA, + ModuleAnalysisManager::Invalidator &Inv); using iterator = MapT::iterator; using const_iterator = MapT::const_iterator; @@ -205,7 +205,7 @@ class CollectorMetadataAnalysis public: using Result = GCStrategyMap; - Result run(Module &M, ModuleAnalysisManager &MAM); + LLVM_ABI Result run(Module &M, ModuleAnalysisManager &MAM); }; /// An analysis pass which caches information about the Function. @@ -217,7 +217,7 @@ class GCFunctionAnalysis : public AnalysisInfoMixin<GCFunctionAnalysis> { public: using Result = GCFunctionInfo; - Result run(Function &F, FunctionAnalysisManager &FAM); + LLVM_ABI Result run(Function &F, FunctionAnalysisManager &FAM); }; /// LowerIntrinsics - This pass rewrites calls to the llvm.gcread or @@ -228,7 +228,7 @@ public: /// This pass requires `CollectorMetadataAnalysis`. class GCLoweringPass : public PassInfoMixin<GCLoweringPass> { public: - PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); + LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); }; /// An analysis pass which caches information about the entire Module. @@ -244,7 +244,7 @@ public: /// Lookup the GCStrategy object associated with the given gc name. /// Objects are owned internally; No caller should attempt to delete the /// returned objects. - GCStrategy *getGCStrategy(const StringRef Name); + LLVM_ABI GCStrategy *getGCStrategy(const StringRef Name); /// List of per function info objects. In theory, Each of these /// may be associated with a different GC. @@ -265,14 +265,14 @@ private: public: using iterator = SmallVector<std::unique_ptr<GCStrategy>, 1>::const_iterator; - static char ID; + LLVM_ABI static char ID; - GCModuleInfo(); + LLVM_ABI GCModuleInfo(); /// clear - Resets the pass. Any pass, which uses GCModuleInfo, should /// call it in doFinalization(). /// - void clear(); + LLVM_ABI void clear(); /// begin/end - Iterators for used strategies. /// @@ -282,7 +282,7 @@ public: /// get - Look up function metadata. This is currently assumed /// have the side effect of initializing the associated GCStrategy. That /// will soon change. - GCFunctionInfo &getFunctionInfo(const Function &F); + LLVM_ABI GCFunctionInfo &getFunctionInfo(const Function &F); }; } // end namespace llvm diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h index da73238..490d1a3 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h @@ -103,6 +103,20 @@ public: /// \return The known alignment for the pointer-like value \p R. Align computeKnownAlignment(Register R, unsigned Depth = 0); + /// If a G_SHL/G_ASHR/G_LSHR node with shift operand \p R has shift amounts + /// that are all less than the element bit-width of the shift node, return the + /// valid constant range. + std::optional<ConstantRange> + getValidShiftAmountRange(Register R, const APInt &DemandedElts, + unsigned Depth); + + /// If a G_SHL/G_ASHR/G_LSHR node with shift operand \p R has shift amounts + /// that are all less than the element bit-width of the shift node, return the + /// minimum possible value. + std::optional<uint64_t> getValidMinimumShiftAmount(Register R, + const APInt &DemandedElts, + unsigned Depth = 0); + /// Determine which floating-point classes are valid for \p V, and return them /// in KnownFPClass bit sets. /// diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h index 571ec6d..4292c0b 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h @@ -26,9 +26,9 @@ namespace llvm { /// A base class for all GenericMachineInstrs. class GenericMachineInstr : public MachineInstr { - constexpr static unsigned PoisonFlags = NoUWrap | NoSWrap | NoUSWrap | - IsExact | Disjoint | NonNeg | - FmNoNans | FmNoInfs | SameSign; + constexpr static unsigned PoisonFlags = + NoUWrap | NoSWrap | NoUSWrap | IsExact | Disjoint | NonNeg | FmNoNans | + FmNoInfs | SameSign | InBounds; public: GenericMachineInstr() = delete; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 756c0b2..99d3cd0 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -518,6 +518,21 @@ public: const SrcOp &Op1, std::optional<unsigned> Flags = std::nullopt); + /// Build and insert an instruction with appropriate flags for addressing some + /// offset of an object, i.e.: \p Res = nuw inbounds G_PTR_ADD \p Op0, \p Op1 + /// The value of \p Op0 must be a pointer into or just after an object, adding + /// the value of \p Op1 to it must yield to a pointer into or just after the + /// same object. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res and \p Op0 must be generic virtual registers with pointer + /// type. + /// \pre \p Op1 must be a generic virtual register with scalar type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildObjectPtrOffset(const DstOp &Res, const SrcOp &Op0, + const SrcOp &Op1); + /// Materialize and insert \p Res = G_PTR_ADD \p Op0, (G_CONSTANT \p Value) /// /// G_PTR_ADD adds \p Value bytes to the pointer specified by \p Op0, @@ -534,10 +549,29 @@ public: /// type as \p Op0 or \p Op0 itself. /// /// \return a MachineInstrBuilder for the newly created instruction. - std::optional<MachineInstrBuilder> materializePtrAdd(Register &Res, - Register Op0, - const LLT ValueTy, - uint64_t Value); + std::optional<MachineInstrBuilder> + materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, + uint64_t Value, + std::optional<unsigned> Flags = std::nullopt); + + /// Materialize and insert an instruction with appropriate flags for + /// addressing some offset of an object, i.e.: + /// \p Res = nuw inbounds G_PTR_ADD \p Op0, (G_CONSTANT \p Value) + /// The value of \p Op0 must be a pointer into or just after an object, adding + /// \p Value to it must yield to a pointer into or just after the same object. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Op0 must be a generic virtual register with pointer type. + /// \pre \p ValueTy must be a scalar type. + /// \pre \p Res must be 0. This is to detect confusion between + /// materializeObjectPtrOffset() and buildObjectPtrOffset(). + /// \post \p Res will either be a new generic virtual register of the same + /// type as \p Op0 or \p Op0 itself. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + std::optional<MachineInstrBuilder> + materializeObjectPtrOffset(Register &Res, Register Op0, const LLT ValueTy, + uint64_t Value); /// Build and insert \p Res = G_PTRMASK \p Op0, \p Op1 MachineInstrBuilder buildPtrMask(const DstOp &Res, const SrcOp &Op0, diff --git a/llvm/include/llvm/CodeGen/LinkAllAsmWriterComponents.h b/llvm/include/llvm/CodeGen/LinkAllAsmWriterComponents.h index c22f9d4..c70413d 100644 --- a/llvm/include/llvm/CodeGen/LinkAllAsmWriterComponents.h +++ b/llvm/include/llvm/CodeGen/LinkAllAsmWriterComponents.h @@ -15,19 +15,17 @@ #define LLVM_CODEGEN_LINKALLASMWRITERCOMPONENTS_H #include "llvm/IR/BuiltinGCs.h" -#include <cstdlib> +#include "llvm/Support/AlwaysTrue.h" namespace { struct ForceAsmWriterLinking { ForceAsmWriterLinking() { // We must reference the plug-ins in such a way that compilers will not // delete it all as dead code, even with whole program optimization, - // yet is effectively a NO-OP. As the compiler isn't smart enough - // to know that getenv() never returns -1, this will do the job. - // This is so that globals in the translation units where these functions - // are defined are forced to be initialized, populating various - // registries. - if (std::getenv("bar") != (char*) -1) + // yet is effectively a NO-OP. This is so that globals in the translation + // units where these functions are defined are forced to be initialized, + // populating various registries. + if (llvm::getNonFoldableAlwaysTrue()) return; llvm::linkOcamlGCPrinter(); diff --git a/llvm/include/llvm/CodeGen/LinkAllCodegenComponents.h b/llvm/include/llvm/CodeGen/LinkAllCodegenComponents.h index 6f56682..f0a01d2 100644 --- a/llvm/include/llvm/CodeGen/LinkAllCodegenComponents.h +++ b/llvm/include/llvm/CodeGen/LinkAllCodegenComponents.h @@ -16,20 +16,18 @@ #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/SchedulerRegistry.h" +#include "llvm/Support/AlwaysTrue.h" #include "llvm/Target/TargetMachine.h" -#include <cstdlib> namespace { struct ForceCodegenLinking { ForceCodegenLinking() { // We must reference the passes in such a way that compilers will not // delete it all as dead code, even with whole program optimization, - // yet is effectively a NO-OP. As the compiler isn't smart enough - // to know that getenv() never returns -1, this will do the job. - // This is so that globals in the translation units where these functions - // are defined are forced to be initialized, populating various - // registries. - if (std::getenv("bar") != (char*) -1) + // yet is effectively a NO-OP. This is so that globals in the translation + // units where these functions are defined are forced to be initialized, + // populating various registries. + if (llvm::getNonFoldableAlwaysTrue()) return; (void) llvm::createFastRegisterAllocator(); diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h index 119786f..0f3f945 100644 --- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h +++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h @@ -482,6 +482,8 @@ struct CallSiteInfo { MachineInstrLoc CallLocation; std::vector<ArgRegPair> ArgForwardingRegs; + /// Numeric callee type identifiers for the callgraph section. + std::vector<uint64_t> CalleeTypeIds; bool operator==(const CallSiteInfo &Other) const { return CallLocation.BlockNum == Other.CallLocation.BlockNum && @@ -511,6 +513,7 @@ template <> struct MappingTraits<CallSiteInfo> { YamlIO.mapRequired("offset", CSInfo.CallLocation.Offset); YamlIO.mapOptional("fwdArgRegs", CSInfo.ArgForwardingRegs, std::vector<CallSiteInfo::ArgRegPair>()); + YamlIO.mapOptional("calleeTypeIds", CSInfo.CalleeTypeIds); } static const bool flow = true; diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h index e5958ec..7f88323 100644 --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -515,6 +515,8 @@ public: struct CallSiteInfo { /// Vector of call argument and its forwarding register. SmallVector<ArgRegPair, 1> ArgRegPairs; + /// Callee type ids. + SmallVector<ConstantInt *, 4> CalleeTypeIds; }; struct CalledGlobalInfo { diff --git a/llvm/include/llvm/CodeGen/MachineFunctionAnalysis.h b/llvm/include/llvm/CodeGen/MachineFunctionAnalysis.h index 1d954cf..1982ac6 100644 --- a/llvm/include/llvm/CodeGen/MachineFunctionAnalysis.h +++ b/llvm/include/llvm/CodeGen/MachineFunctionAnalysis.h @@ -48,7 +48,7 @@ public: class FreeMachineFunctionPass : public PassInfoMixin<FreeMachineFunctionPass> { public: - PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); + LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); }; } // namespace llvm diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h index 94d04b8..10a9b1f 100644 --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -122,7 +122,9 @@ public: Disjoint = 1 << 19, // Each bit is zero in at least one of the inputs. NoUSWrap = 1 << 20, // Instruction supports geps // no unsigned signed wrap. - SameSign = 1 << 21 // Both operands have the same sign. + SameSign = 1 << 21, // Both operands have the same sign. + InBounds = 1 << 22 // Pointer arithmetic remains inbounds. + // Implies NoUSWrap. }; private: diff --git a/llvm/include/llvm/CodeGen/MachineInstrBundle.h b/llvm/include/llvm/CodeGen/MachineInstrBundle.h index d324236..65eb5c4 100644 --- a/llvm/include/llvm/CodeGen/MachineInstrBundle.h +++ b/llvm/include/llvm/CodeGen/MachineInstrBundle.h @@ -15,6 +15,7 @@ #define LLVM_CODEGEN_MACHINEINSTRBUNDLE_H #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachinePassManager.h" #include "llvm/Support/Compiler.h" namespace llvm { @@ -294,6 +295,12 @@ LLVM_ABI PhysRegInfo AnalyzePhysRegInBundle(const MachineInstr &MI, Register Reg, const TargetRegisterInfo *TRI); +class FinalizeBundleTestPass : public PassInfoMixin<FinalizeBundleTestPass> { +public: + LLVM_ABI PreservedAnalyses run(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM); +}; + } // End llvm namespace #endif diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h index e7a7091..efda7eb 100644 --- a/llvm/include/llvm/CodeGen/MachineScheduler.h +++ b/llvm/include/llvm/CodeGen/MachineScheduler.h @@ -65,7 +65,7 @@ // // void <SubTarget>Subtarget:: // overrideSchedPolicy(MachineSchedPolicy &Policy, -// unsigned NumRegionInstrs) const { +// const SchedRegion &Region) const { // Policy.<Flag> = true; // } // @@ -218,6 +218,22 @@ struct MachineSchedPolicy { MachineSchedPolicy() = default; }; +/// A region of an MBB for scheduling. +struct SchedRegion { + /// RegionBegin is the first instruction in the scheduling region, and + /// RegionEnd is either MBB->end() or the scheduling boundary after the + /// last instruction in the scheduling region. These iterators cannot refer + /// to instructions outside of the identified scheduling region because + /// those may be reordered before scheduling this region. + MachineBasicBlock::iterator RegionBegin; + MachineBasicBlock::iterator RegionEnd; + unsigned NumRegionInstrs; + + SchedRegion(MachineBasicBlock::iterator B, MachineBasicBlock::iterator E, + unsigned N) + : RegionBegin(B), RegionEnd(E), NumRegionInstrs(N) {} +}; + /// MachineSchedStrategy - Interface to the scheduling algorithm used by /// ScheduleDAGMI. /// diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h index 714285e..095a40e 100644 --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -438,10 +438,6 @@ LLVM_ABI extern char &UnpackMachineBundlesID; LLVM_ABI FunctionPass * createUnpackMachineBundles(std::function<bool(const MachineFunction &)> Ftor); -/// FinalizeMachineBundles - This pass finalize machine instruction -/// bundles (created earlier, e.g. during pre-RA scheduling). -LLVM_ABI extern char &FinalizeMachineBundlesID; - /// StackMapLiveness - This pass analyses the register live-out set of /// stackmap/patchpoint intrinsics and attaches the calculated information to /// the intrinsic for later emission to the StackMap. diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 657951d..e5644a5 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1202,13 +1202,16 @@ public: LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops, const SDNodeFlags Flags); LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, - ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops); + ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops, + const SDNodeFlags Flags); LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, ArrayRef<SDValue> Ops, const SDNodeFlags Flags); // Use flags from current flag inserter. LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops); + LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, + ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops); LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, ArrayRef<SDValue> Ops); LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, @@ -1346,9 +1349,10 @@ public: /// Helper function to make it easier to build SelectCC's if you just have an /// ISD::CondCode instead of an SDValue. SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, - SDValue False, ISD::CondCode Cond) { + SDValue False, ISD::CondCode Cond, + SDNodeFlags Flags = SDNodeFlags()) { return getNode(ISD::SELECT_CC, DL, True.getValueType(), LHS, RHS, True, - False, getCondCode(Cond)); + False, getCondCode(Cond), Flags); } /// Try to simplify a select/vselect into 1 of its operands or a constant. @@ -1424,11 +1428,9 @@ public: EVT MemVT, MachineMemOperand *MMO); /// Creates a LifetimeSDNode that starts (`IsStart==true`) or ends - /// (`IsStart==false`) the lifetime of the portion of `FrameIndex` between - /// offsets `Offset` and `Offset + Size`. + /// (`IsStart==false`) the lifetime of the `FrameIndex`. LLVM_ABI SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, - int FrameIndex, int64_t Size, - int64_t Offset = -1); + int FrameIndex); /// Creates a PseudoProbeSDNode with function GUID `Guid` and /// the index of the block `Index` it is probing, as well as the attributes diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 5d9937f..8f88811 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1999,31 +1999,19 @@ public: } }; -/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate -/// the offet and size that are started/ended in the underlying FrameIndex. +/// This SDNode is used for LIFETIME_START/LIFETIME_END values. class LifetimeSDNode : public SDNode { friend class SelectionDAG; - int64_t Size; - int64_t Offset; // -1 if offset is unknown. LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, - SDVTList VTs, int64_t Size, int64_t Offset) - : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {} + SDVTList VTs) + : SDNode(Opcode, Order, dl, VTs) {} + public: int64_t getFrameIndex() const { return cast<FrameIndexSDNode>(getOperand(1))->getIndex(); } - bool hasOffset() const { return Offset >= 0; } - int64_t getOffset() const { - assert(hasOffset() && "offset is unknown"); - return Offset; - } - int64_t getSize() const { - assert(hasOffset() && "offset is unknown"); - return Size; - } - // Methods to support isa and dyn_cast static bool classof(const SDNode *N) { return N->getOpcode() == ISD::LIFETIME_START || diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 1a548a5..cbdc1b6 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -3219,25 +3219,19 @@ public: /// Lower an interleaved store to target specific intrinsics. Return /// true on success. /// - /// \p SI is the vector store instruction. + /// \p SI is the vector store instruction. Can be either a plain store + /// or a vp.store. + /// \p Mask is a per-segment (i.e. number of lanes equal to that of one + /// component being interwoven) mask. Can be nullptr, in which case the + /// result is unconditional. /// \p SVI is the shufflevector to RE-interleave the stored vector. /// \p Factor is the interleave factor. - virtual bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, + virtual bool lowerInterleavedStore(Instruction *Store, Value *Mask, + ShuffleVectorInst *SVI, unsigned Factor) const { return false; } - /// Lower an interleaved store to target specific intrinsics. Return - /// true on success. - /// - /// \p Store is the vp.store instruction. - /// \p Mask is a mask value - /// \p InterleaveOps is a list of values being interleaved. - virtual bool lowerInterleavedVPStore(VPIntrinsic *Store, Value *Mask, - ArrayRef<Value *> InterleaveOps) const { - return false; - } - /// Lower a deinterleave intrinsic to a target specific load intrinsic. /// Return true on success. Currently only supports /// llvm.vector.deinterleave{2,3,5,7} diff --git a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h index 45e67d8..a8c7a8a 100644 --- a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h +++ b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h @@ -54,6 +54,7 @@ class TargetRegisterClass; class TargetRegisterInfo; class TargetSchedModel; class Triple; +struct SchedRegion; //===----------------------------------------------------------------------===// /// @@ -231,7 +232,7 @@ public: /// scheduling heuristics (no custom MachineSchedStrategy) to make /// changes to the generic scheduling policy. virtual void overrideSchedPolicy(MachineSchedPolicy &Policy, - unsigned NumRegionInstrs) const {} + const SchedRegion &Region) const {} /// Override generic post-ra scheduling policy within a region. /// @@ -241,7 +242,7 @@ public: /// Note that some options like tracking register pressure won't take effect /// in post-ra scheduling. virtual void overridePostRASchedPolicy(MachineSchedPolicy &Policy, - unsigned NumRegionInstrs) const {} + const SchedRegion &Region) const {} // Perform target-specific adjustments to the latency of a schedule // dependency. diff --git a/llvm/include/llvm/Config/abi-breaking.h.cmake b/llvm/include/llvm/Config/abi-breaking.h.cmake index 2d27e02..330f360 100644 --- a/llvm/include/llvm/Config/abi-breaking.h.cmake +++ b/llvm/include/llvm/Config/abi-breaking.h.cmake @@ -12,12 +12,41 @@ #ifndef LLVM_ABI_BREAKING_CHECKS_H #define LLVM_ABI_BREAKING_CHECKS_H +// llvm-config.h is required for LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS +#include "llvm/Config/llvm-config.h" + /* Define to enable checks that alter the LLVM C++ ABI */ #cmakedefine01 LLVM_ENABLE_ABI_BREAKING_CHECKS /* Define to enable reverse iteration of unordered llvm containers */ #cmakedefine01 LLVM_ENABLE_REVERSE_ITERATION +#if !defined(__has_attribute) +#define __has_attribute(attribute) 0 +#endif + +// Properly annotate EnableABIBreakingChecks or DisableABIBreakingChecks for +// export from shared library. +// TODO(https://github.com/llvm/llvm-project/issues/145406): eliminate need for +// two preprocessor definitions to gate LLVM_ABI macro definitions. +#if defined(LLVM_BUILD_STATIC) || !defined(LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS) +#define ABI_BREAKING_EXPORT_ABI +#else +#if defined(_WIN32) +#if defined(LLVM_EXPORTS) +#define ABI_BREAKING_EXPORT_ABI __declspec(dllexport) +#else +#define ABI_BREAKING_EXPORT_ABI __declspec(dllimport) +#endif +#else +#if __has_attribute(visibility) +#define ABI_BREAKING_EXPORT_ABI __attribute__((__visibility__("default"))) +#else +#define ABI_BREAKING_EXPORT_ABI +#endif +#endif +#endif + /* Allow selectively disabling link-time mismatch checking so that header-only ADT content from LLVM can be used without linking libSupport. */ #if !defined(LLVM_DISABLE_ABI_BREAKING_CHECKS_ENFORCING) || !LLVM_DISABLE_ABI_BREAKING_CHECKS_ENFORCING @@ -43,12 +72,12 @@ #endif namespace llvm { #if LLVM_ENABLE_ABI_BREAKING_CHECKS -extern int EnableABIBreakingChecks; +ABI_BREAKING_EXPORT_ABI extern int EnableABIBreakingChecks; LLVM_HIDDEN_VISIBILITY __attribute__((weak)) int *VerifyEnableABIBreakingChecks = &EnableABIBreakingChecks; #else -extern int DisableABIBreakingChecks; +ABI_BREAKING_EXPORT_ABI extern int DisableABIBreakingChecks; LLVM_HIDDEN_VISIBILITY __attribute__((weak)) int *VerifyDisableABIBreakingChecks = &DisableABIBreakingChecks; diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake index a683229..39136bc 100644 --- a/llvm/include/llvm/Config/llvm-config.h.cmake +++ b/llvm/include/llvm/Config/llvm-config.h.cmake @@ -101,6 +101,9 @@ /* Define if LLVM is using tflite */ #cmakedefine LLVM_HAVE_TFLITE +/* Define if we want to check profile consistency in lit tests */ +#cmakedefine LLVM_ENABLE_PROFCHECK + /* Define to 1 if you have the <sysexits.h> header file. */ #cmakedefine HAVE_SYSEXITS_H ${HAVE_SYSEXITS_H} diff --git a/llvm/include/llvm/Demangle/Demangle.h b/llvm/include/llvm/Demangle/Demangle.h index 21e7457..d9b08b2 100644 --- a/llvm/include/llvm/Demangle/Demangle.h +++ b/llvm/include/llvm/Demangle/Demangle.h @@ -9,6 +9,7 @@ #ifndef LLVM_DEMANGLE_DEMANGLE_H #define LLVM_DEMANGLE_DEMANGLE_H +#include "DemangleConfig.h" #include <cstddef> #include <optional> #include <string> @@ -33,7 +34,8 @@ enum : int { /// Returns a non-NULL pointer to a NUL-terminated C style string /// that should be explicitly freed, if successful. Otherwise, may return /// nullptr if mangled_name is not a valid mangling or is nullptr. -char *itaniumDemangle(std::string_view mangled_name, bool ParseParams = true); +DEMANGLE_ABI char *itaniumDemangle(std::string_view mangled_name, + bool ParseParams = true); enum MSDemangleFlags { MSDF_None = 0, @@ -52,87 +54,90 @@ enum MSDemangleFlags { /// bytes of the input string were consumed. /// status receives one of the demangle_ enum entries above if it's not nullptr. /// Flags controls various details of the demangled representation. -char *microsoftDemangle(std::string_view mangled_name, size_t *n_read, - int *status, MSDemangleFlags Flags = MSDF_None); +DEMANGLE_ABI char *microsoftDemangle(std::string_view mangled_name, + size_t *n_read, int *status, + MSDemangleFlags Flags = MSDF_None); -std::optional<size_t> +DEMANGLE_ABI std::optional<size_t> getArm64ECInsertionPointInMangledName(std::string_view MangledName); // Demangles a Rust v0 mangled symbol. -char *rustDemangle(std::string_view MangledName); +DEMANGLE_ABI char *rustDemangle(std::string_view MangledName); // Demangles a D mangled symbol. -char *dlangDemangle(std::string_view MangledName); +DEMANGLE_ABI char *dlangDemangle(std::string_view MangledName); /// Attempt to demangle a string using different demangling schemes. /// The function uses heuristics to determine which demangling scheme to use. /// \param MangledName - reference to string to demangle. /// \returns - the demangled string, or a copy of the input string if no /// demangling occurred. -std::string demangle(std::string_view MangledName); +DEMANGLE_ABI std::string demangle(std::string_view MangledName); -bool nonMicrosoftDemangle(std::string_view MangledName, std::string &Result, - bool CanHaveLeadingDot = true, - bool ParseParams = true); +DEMANGLE_ABI bool nonMicrosoftDemangle(std::string_view MangledName, + std::string &Result, + bool CanHaveLeadingDot = true, + bool ParseParams = true); /// "Partial" demangler. This supports demangling a string into an AST /// (typically an intermediate stage in itaniumDemangle) and querying certain /// properties or partially printing the demangled name. struct ItaniumPartialDemangler { - ItaniumPartialDemangler(); + DEMANGLE_ABI ItaniumPartialDemangler(); - ItaniumPartialDemangler(ItaniumPartialDemangler &&Other); - ItaniumPartialDemangler &operator=(ItaniumPartialDemangler &&Other); + DEMANGLE_ABI ItaniumPartialDemangler(ItaniumPartialDemangler &&Other); + DEMANGLE_ABI ItaniumPartialDemangler & + operator=(ItaniumPartialDemangler &&Other); /// Demangle into an AST. Subsequent calls to the rest of the member functions /// implicitly operate on the AST this produces. /// \return true on error, false otherwise - bool partialDemangle(const char *MangledName); + DEMANGLE_ABI bool partialDemangle(const char *MangledName); /// Just print the entire mangled name into Buf. Buf and N behave like the /// second and third parameters to __cxa_demangle. - char *finishDemangle(char *Buf, size_t *N) const; + DEMANGLE_ABI char *finishDemangle(char *Buf, size_t *N) const; /// See \ref finishDemangle /// /// \param[in] OB A llvm::itanium_demangle::OutputBuffer that the demangled /// name will be printed into. /// - char *finishDemangle(void *OB) const; + DEMANGLE_ABI char *finishDemangle(void *OB) const; /// Get the base name of a function. This doesn't include trailing template /// arguments, ie for "a::b<int>" this function returns "b". - char *getFunctionBaseName(char *Buf, size_t *N) const; + DEMANGLE_ABI char *getFunctionBaseName(char *Buf, size_t *N) const; /// Get the context name for a function. For "a::b::c", this function returns /// "a::b". - char *getFunctionDeclContextName(char *Buf, size_t *N) const; + DEMANGLE_ABI char *getFunctionDeclContextName(char *Buf, size_t *N) const; /// Get the entire name of this function. - char *getFunctionName(char *Buf, size_t *N) const; + DEMANGLE_ABI char *getFunctionName(char *Buf, size_t *N) const; /// Get the parameters for this function. - char *getFunctionParameters(char *Buf, size_t *N) const; - char *getFunctionReturnType(char *Buf, size_t *N) const; + DEMANGLE_ABI char *getFunctionParameters(char *Buf, size_t *N) const; + DEMANGLE_ABI char *getFunctionReturnType(char *Buf, size_t *N) const; /// If this function has any cv or reference qualifiers. These imply that /// the function is a non-static member function. - bool hasFunctionQualifiers() const; + DEMANGLE_ABI bool hasFunctionQualifiers() const; /// If this symbol describes a constructor or destructor. - bool isCtorOrDtor() const; + DEMANGLE_ABI bool isCtorOrDtor() const; /// If this symbol describes a function. - bool isFunction() const; + DEMANGLE_ABI bool isFunction() const; /// If this symbol describes a variable. - bool isData() const; + DEMANGLE_ABI bool isData() const; /// If this symbol is a <special-name>. These are generally implicitly /// generated by the implementation, such as vtables and typeinfo names. - bool isSpecialName() const; + DEMANGLE_ABI bool isSpecialName() const; - ~ItaniumPartialDemangler(); + DEMANGLE_ABI ~ItaniumPartialDemangler(); private: void *RootNode; diff --git a/llvm/include/llvm/Demangle/DemangleConfig.h b/llvm/include/llvm/Demangle/DemangleConfig.h index 30f72ff..912c9b8 100644 --- a/llvm/include/llvm/Demangle/DemangleConfig.h +++ b/llvm/include/llvm/Demangle/DemangleConfig.h @@ -15,6 +15,9 @@ #ifndef LLVM_DEMANGLE_DEMANGLECONFIG_H #define LLVM_DEMANGLE_DEMANGLECONFIG_H +// llvm-config.h is required for LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS +#include "llvm/Config/llvm-config.h" + #ifndef __has_feature #define __has_feature(x) 0 #endif @@ -94,4 +97,24 @@ #define DEMANGLE_NAMESPACE_BEGIN namespace llvm { namespace itanium_demangle { #define DEMANGLE_NAMESPACE_END } } +/// DEMANGLE_ABI is the export/visibility macro used to mark symbols delcared in +/// llvm/Demangle as exported when built as a shared library. +#if defined(LLVM_BUILD_STATIC) || !defined(LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS) +#define DEMANGLE_ABI +#else +#if defined(_WIN32) +#if defined(LLVM_EXPORTS) +#define DEMANGLE_ABI __declspec(dllexport) +#else +#define DEMANGLE_ABI __declspec(dllimport) +#endif +#else +#if __has_attribute(visibility) +#define DEMANGLE_ABI __attribute__((__visibility__("default"))) +#else +#define DEMANGLE_ABI +#endif +#endif +#endif + #endif diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h index 5533652..62d427c 100644 --- a/llvm/include/llvm/Demangle/ItaniumDemangle.h +++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h @@ -3049,7 +3049,8 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { Node *parse(bool ParseParams = true); }; -const char* parse_discriminator(const char* first, const char* last); +DEMANGLE_ABI const char *parse_discriminator(const char *first, + const char *last); // <name> ::= <nested-name> // N // ::= <local-name> # See Scope Encoding below // Z diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangle.h b/llvm/include/llvm/Demangle/MicrosoftDemangle.h index b9a25e3..a2af875 100644 --- a/llvm/include/llvm/Demangle/MicrosoftDemangle.h +++ b/llvm/include/llvm/Demangle/MicrosoftDemangle.h @@ -10,6 +10,7 @@ #define LLVM_DEMANGLE_MICROSOFTDEMANGLE_H #include "llvm/Demangle/Demangle.h" +#include "llvm/Demangle/DemangleConfig.h" #include "llvm/Demangle/MicrosoftDemangleNodes.h" #include <cassert> @@ -151,14 +152,14 @@ public: // You are supposed to call parse() first and then check if error is true. If // it is false, call output() to write the formatted name to the given stream. - SymbolNode *parse(std::string_view &MangledName); + DEMANGLE_ABI SymbolNode *parse(std::string_view &MangledName); - TagTypeNode *parseTagUniqueName(std::string_view &MangledName); + DEMANGLE_ABI TagTypeNode *parseTagUniqueName(std::string_view &MangledName); // True if an error occurred. bool Error = false; - void dumpBackReferences(); + DEMANGLE_ABI void dumpBackReferences(); private: SymbolNode *demangleEncodedSymbol(std::string_view &MangledName, diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h index a9cfe72..155cfe8 100644 --- a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h +++ b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h @@ -13,6 +13,7 @@ #ifndef LLVM_DEMANGLE_MICROSOFTDEMANGLENODES_H #define LLVM_DEMANGLE_MICROSOFTDEMANGLENODES_H +#include "DemangleConfig.h" #include <array> #include <cstdint> #include <string> @@ -281,7 +282,7 @@ struct Node { virtual void output(OutputBuffer &OB, OutputFlags Flags) const = 0; - std::string toString(OutputFlags Flags = OF_Default) const; + DEMANGLE_ABI std::string toString(OutputFlags Flags = OF_Default) const; private: NodeKind Kind; @@ -332,7 +333,7 @@ struct TypeNode : public Node { Qualifiers Quals = Q_None; }; -struct PrimitiveTypeNode : public TypeNode { +struct DEMANGLE_ABI PrimitiveTypeNode : public TypeNode { explicit PrimitiveTypeNode(PrimitiveKind K) : TypeNode(NodeKind::PrimitiveType), PrimKind(K) {} @@ -346,7 +347,7 @@ struct PrimitiveTypeNode : public TypeNode { PrimitiveKind PrimKind; }; -struct FunctionSignatureNode : public TypeNode { +struct DEMANGLE_ABI FunctionSignatureNode : public TypeNode { explicit FunctionSignatureNode(NodeKind K) : TypeNode(K) {} FunctionSignatureNode() : TypeNode(NodeKind::FunctionSignature) {} @@ -394,10 +395,11 @@ struct IdentifierNode : public Node { NodeArrayNode *TemplateParams = nullptr; protected: - void outputTemplateParameters(OutputBuffer &OB, OutputFlags Flags) const; + DEMANGLE_ABI void outputTemplateParameters(OutputBuffer &OB, + OutputFlags Flags) const; }; -struct VcallThunkIdentifierNode : public IdentifierNode { +struct DEMANGLE_ABI VcallThunkIdentifierNode : public IdentifierNode { VcallThunkIdentifierNode() : IdentifierNode(NodeKind::VcallThunkIdentifier) {} void output(OutputBuffer &OB, OutputFlags Flags) const override; @@ -409,7 +411,7 @@ struct VcallThunkIdentifierNode : public IdentifierNode { uint64_t OffsetInVTable = 0; }; -struct DynamicStructorIdentifierNode : public IdentifierNode { +struct DEMANGLE_ABI DynamicStructorIdentifierNode : public IdentifierNode { DynamicStructorIdentifierNode() : IdentifierNode(NodeKind::DynamicStructorIdentifier) {} @@ -424,7 +426,7 @@ struct DynamicStructorIdentifierNode : public IdentifierNode { bool IsDestructor = false; }; -struct NamedIdentifierNode : public IdentifierNode { +struct DEMANGLE_ABI NamedIdentifierNode : public IdentifierNode { NamedIdentifierNode() : IdentifierNode(NodeKind::NamedIdentifier) {} void output(OutputBuffer &OB, OutputFlags Flags) const override; @@ -436,7 +438,7 @@ struct NamedIdentifierNode : public IdentifierNode { std::string_view Name; }; -struct IntrinsicFunctionIdentifierNode : public IdentifierNode { +struct DEMANGLE_ABI IntrinsicFunctionIdentifierNode : public IdentifierNode { explicit IntrinsicFunctionIdentifierNode(IntrinsicFunctionKind Operator) : IdentifierNode(NodeKind::IntrinsicFunctionIdentifier), Operator(Operator) {} @@ -450,7 +452,7 @@ struct IntrinsicFunctionIdentifierNode : public IdentifierNode { IntrinsicFunctionKind Operator; }; -struct LiteralOperatorIdentifierNode : public IdentifierNode { +struct DEMANGLE_ABI LiteralOperatorIdentifierNode : public IdentifierNode { LiteralOperatorIdentifierNode() : IdentifierNode(NodeKind::LiteralOperatorIdentifier) {} @@ -463,7 +465,7 @@ struct LiteralOperatorIdentifierNode : public IdentifierNode { std::string_view Name; }; -struct LocalStaticGuardIdentifierNode : public IdentifierNode { +struct DEMANGLE_ABI LocalStaticGuardIdentifierNode : public IdentifierNode { LocalStaticGuardIdentifierNode() : IdentifierNode(NodeKind::LocalStaticGuardIdentifier) {} @@ -477,7 +479,7 @@ struct LocalStaticGuardIdentifierNode : public IdentifierNode { uint32_t ScopeIndex = 0; }; -struct ConversionOperatorIdentifierNode : public IdentifierNode { +struct DEMANGLE_ABI ConversionOperatorIdentifierNode : public IdentifierNode { ConversionOperatorIdentifierNode() : IdentifierNode(NodeKind::ConversionOperatorIdentifier) {} @@ -491,7 +493,7 @@ struct ConversionOperatorIdentifierNode : public IdentifierNode { TypeNode *TargetType = nullptr; }; -struct StructorIdentifierNode : public IdentifierNode { +struct DEMANGLE_ABI StructorIdentifierNode : public IdentifierNode { StructorIdentifierNode() : IdentifierNode(NodeKind::StructorIdentifier) {} explicit StructorIdentifierNode(bool IsDestructor) : IdentifierNode(NodeKind::StructorIdentifier), @@ -508,7 +510,7 @@ struct StructorIdentifierNode : public IdentifierNode { bool IsDestructor = false; }; -struct ThunkSignatureNode : public FunctionSignatureNode { +struct DEMANGLE_ABI ThunkSignatureNode : public FunctionSignatureNode { ThunkSignatureNode() : FunctionSignatureNode(NodeKind::ThunkSignature) {} void outputPre(OutputBuffer &OB, OutputFlags Flags) const override; @@ -528,7 +530,7 @@ struct ThunkSignatureNode : public FunctionSignatureNode { ThisAdjustor ThisAdjust; }; -struct PointerTypeNode : public TypeNode { +struct DEMANGLE_ABI PointerTypeNode : public TypeNode { PointerTypeNode() : TypeNode(NodeKind::PointerType) {} void outputPre(OutputBuffer &OB, OutputFlags Flags) const override; void outputPost(OutputBuffer &OB, OutputFlags Flags) const override; @@ -550,7 +552,7 @@ struct PointerTypeNode : public TypeNode { TypeNode *Pointee = nullptr; }; -struct TagTypeNode : public TypeNode { +struct DEMANGLE_ABI TagTypeNode : public TypeNode { explicit TagTypeNode(TagKind Tag) : TypeNode(NodeKind::TagType), Tag(Tag) {} void outputPre(OutputBuffer &OB, OutputFlags Flags) const override; @@ -562,7 +564,7 @@ struct TagTypeNode : public TypeNode { TagKind Tag; }; -struct ArrayTypeNode : public TypeNode { +struct DEMANGLE_ABI ArrayTypeNode : public TypeNode { ArrayTypeNode() : TypeNode(NodeKind::ArrayType) {} void outputPre(OutputBuffer &OB, OutputFlags Flags) const override; @@ -591,7 +593,7 @@ struct IntrinsicNode : public TypeNode { } }; -struct CustomTypeNode : public TypeNode { +struct DEMANGLE_ABI CustomTypeNode : public TypeNode { CustomTypeNode() : TypeNode(NodeKind::Custom) {} void outputPre(OutputBuffer &OB, OutputFlags Flags) const override; @@ -602,7 +604,7 @@ struct CustomTypeNode : public TypeNode { IdentifierNode *Identifier = nullptr; }; -struct NodeArrayNode : public Node { +struct DEMANGLE_ABI NodeArrayNode : public Node { NodeArrayNode() : Node(NodeKind::NodeArray) {} void output(OutputBuffer &OB, OutputFlags Flags) const override; @@ -618,7 +620,7 @@ struct NodeArrayNode : public Node { size_t Count = 0; }; -struct QualifiedNameNode : public Node { +struct DEMANGLE_ABI QualifiedNameNode : public Node { QualifiedNameNode() : Node(NodeKind::QualifiedName) {} void output(OutputBuffer &OB, OutputFlags Flags) const override; @@ -635,7 +637,7 @@ struct QualifiedNameNode : public Node { } }; -struct TemplateParameterReferenceNode : public Node { +struct DEMANGLE_ABI TemplateParameterReferenceNode : public Node { TemplateParameterReferenceNode() : Node(NodeKind::TemplateParameterReference) {} @@ -653,7 +655,7 @@ struct TemplateParameterReferenceNode : public Node { bool IsMemberPointer = false; }; -struct IntegerLiteralNode : public Node { +struct DEMANGLE_ABI IntegerLiteralNode : public Node { IntegerLiteralNode() : Node(NodeKind::IntegerLiteral) {} IntegerLiteralNode(uint64_t Value, bool IsNegative) : Node(NodeKind::IntegerLiteral), Value(Value), IsNegative(IsNegative) {} @@ -668,7 +670,7 @@ struct IntegerLiteralNode : public Node { bool IsNegative = false; }; -struct RttiBaseClassDescriptorNode : public IdentifierNode { +struct DEMANGLE_ABI RttiBaseClassDescriptorNode : public IdentifierNode { RttiBaseClassDescriptorNode() : IdentifierNode(NodeKind::RttiBaseClassDescriptor) {} @@ -684,7 +686,7 @@ struct RttiBaseClassDescriptorNode : public IdentifierNode { uint32_t Flags = 0; }; -struct SymbolNode : public Node { +struct DEMANGLE_ABI SymbolNode : public Node { explicit SymbolNode(NodeKind K) : Node(K) {} void output(OutputBuffer &OB, OutputFlags Flags) const override; @@ -696,7 +698,7 @@ struct SymbolNode : public Node { QualifiedNameNode *Name = nullptr; }; -struct SpecialTableSymbolNode : public SymbolNode { +struct DEMANGLE_ABI SpecialTableSymbolNode : public SymbolNode { explicit SpecialTableSymbolNode() : SymbolNode(NodeKind::SpecialTableSymbol) {} @@ -710,7 +712,7 @@ struct SpecialTableSymbolNode : public SymbolNode { Qualifiers Quals = Qualifiers::Q_None; }; -struct LocalStaticGuardVariableNode : public SymbolNode { +struct DEMANGLE_ABI LocalStaticGuardVariableNode : public SymbolNode { LocalStaticGuardVariableNode() : SymbolNode(NodeKind::LocalStaticGuardVariable) {} @@ -723,7 +725,7 @@ struct LocalStaticGuardVariableNode : public SymbolNode { bool IsVisible = false; }; -struct EncodedStringLiteralNode : public SymbolNode { +struct DEMANGLE_ABI EncodedStringLiteralNode : public SymbolNode { EncodedStringLiteralNode() : SymbolNode(NodeKind::EncodedStringLiteral) {} void output(OutputBuffer &OB, OutputFlags Flags) const override; @@ -737,7 +739,7 @@ struct EncodedStringLiteralNode : public SymbolNode { CharKind Char = CharKind::Char; }; -struct VariableSymbolNode : public SymbolNode { +struct DEMANGLE_ABI VariableSymbolNode : public SymbolNode { VariableSymbolNode() : SymbolNode(NodeKind::VariableSymbol) {} void output(OutputBuffer &OB, OutputFlags Flags) const override; @@ -750,7 +752,7 @@ struct VariableSymbolNode : public SymbolNode { TypeNode *Type = nullptr; }; -struct FunctionSymbolNode : public SymbolNode { +struct DEMANGLE_ABI FunctionSymbolNode : public SymbolNode { FunctionSymbolNode() : SymbolNode(NodeKind::FunctionSymbol) {} void output(OutputBuffer &OB, OutputFlags Flags) const override; @@ -762,7 +764,7 @@ struct FunctionSymbolNode : public SymbolNode { FunctionSignatureNode *Signature = nullptr; }; -struct PointerAuthQualifierNode : public Node { +struct DEMANGLE_ABI PointerAuthQualifierNode : public Node { PointerAuthQualifierNode() : Node(NodeKind::PointerAuthQualifier) {} // __ptrauth takes three arguments: diff --git a/llvm/include/llvm/ExecutionEngine/MCJIT.h b/llvm/include/llvm/ExecutionEngine/MCJIT.h index c836c06..1e035c0 100644 --- a/llvm/include/llvm/ExecutionEngine/MCJIT.h +++ b/llvm/include/llvm/ExecutionEngine/MCJIT.h @@ -15,8 +15,8 @@ #define LLVM_EXECUTIONENGINE_MCJIT_H #include "llvm/ExecutionEngine/ExecutionEngine.h" +#include "llvm/Support/AlwaysTrue.h" #include "llvm/Support/Compiler.h" -#include <cstdlib> extern "C" LLVM_ABI void LLVMLinkInMCJIT(); @@ -24,13 +24,11 @@ namespace { struct ForceMCJITLinking { ForceMCJITLinking() { // We must reference MCJIT in such a way that compilers will not - // delete it all as dead code, even with whole program optimization, - // yet is effectively a NO-OP. As the compiler isn't smart enough - // to know that getenv() never returns -1, this will do the job. - // This is so that globals in the translation units where these functions - // are defined are forced to be initialized, populating various - // registries. - if (std::getenv("bar") != (char*) -1) + // delete it all as dead code, even with whole program optimization, yet + // is effectively a NO-OP. This is so that globals in the translation + // units where these functions are defined are forced to be initialized, + // populating various registries. + if (llvm::getNonFoldableAlwaysTrue()) return; LLVMLinkInMCJIT(); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h index 2834331..b865e02 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h @@ -17,6 +17,7 @@ #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" #include "llvm/ExecutionEngine/Orc/LazyReexports.h" +#include "llvm/Support/Compiler.h" #include <mutex> @@ -33,7 +34,7 @@ class EPCIndirectionUtils { public: /// ABI support base class. Used to write resolver, stub, and trampoline /// blocks. - class ABISupport { + class LLVM_ABI ABISupport { protected: ABISupport(unsigned PointerSize, unsigned TrampolineSize, unsigned StubSize, unsigned StubToPointerMaxDisplacement, unsigned ResolverCodeSize) @@ -81,7 +82,7 @@ public: CreateWithABI(ExecutorProcessControl &EPC); /// Create based on the ExecutorProcessControl triple. - static Expected<std::unique_ptr<EPCIndirectionUtils>> + LLVM_ABI static Expected<std::unique_ptr<EPCIndirectionUtils>> Create(ExecutorProcessControl &EPC); /// Create based on the ExecutorProcessControl triple. @@ -98,27 +99,27 @@ public: /// Release memory for resources held by this instance. This *must* be called /// prior to destruction of the class. - Error cleanup(); + LLVM_ABI Error cleanup(); /// Write resolver code to the executor process and return its address. /// This must be called before any call to createTrampolinePool or /// createLazyCallThroughManager. - Expected<ExecutorAddr> writeResolverBlock(ExecutorAddr ReentryFnAddr, - ExecutorAddr ReentryCtxAddr); + LLVM_ABI Expected<ExecutorAddr> + writeResolverBlock(ExecutorAddr ReentryFnAddr, ExecutorAddr ReentryCtxAddr); /// Returns the address of the Resolver block. Returns zero if the /// writeResolverBlock method has not previously been called. ExecutorAddr getResolverBlockAddress() const { return ResolverBlockAddr; } /// Create an IndirectStubsManager for the executor process. - std::unique_ptr<IndirectStubsManager> createIndirectStubsManager(); + LLVM_ABI std::unique_ptr<IndirectStubsManager> createIndirectStubsManager(); /// Create a TrampolinePool for the executor process. - TrampolinePool &getTrampolinePool(); + LLVM_ABI TrampolinePool &getTrampolinePool(); /// Create a LazyCallThroughManager. /// This function should only be called once. - LazyCallThroughManager & + LLVM_ABI LazyCallThroughManager & createLazyCallThroughManager(ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr); @@ -170,7 +171,7 @@ private: /// called. /// /// This function is experimental and likely subject to revision. -Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU); +LLVM_ABI Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU); namespace detail { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h b/llvm/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h index c92719e..a9f5c45 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h @@ -16,6 +16,7 @@ #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/Speculation.h" +#include "llvm/Support/Compiler.h" namespace llvm { @@ -24,8 +25,8 @@ namespace orc { // Provides common code. class SpeculateQuery { protected: - void findCalles(const BasicBlock *, DenseSet<StringRef> &); - bool isStraightLine(const Function &F); + LLVM_ABI void findCalles(const BasicBlock *, DenseSet<StringRef> &); + LLVM_ABI bool isStraightLine(const Function &F); public: using ResultTy = std::optional<DenseMap<StringRef, DenseSet<StringRef>>>; @@ -37,7 +38,7 @@ class BlockFreqQuery : public SpeculateQuery { public: // Find likely next executables based on IR Block Frequency - ResultTy operator()(Function &F); + LLVM_ABI ResultTy operator()(Function &F); }; // This Query generates a sequence of basic blocks which follows the order of @@ -73,7 +74,7 @@ private: VisitedBlocksInfoTy &); public: - ResultTy operator()(Function &F); + LLVM_ABI ResultTy operator()(Function &F); }; } // namespace orc diff --git a/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h b/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h index 0aa122f..6fa51ed 100644 --- a/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h +++ b/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h @@ -15,6 +15,8 @@ #define LLVM_FRONTEND_HLSL_ROOTSIGNATUREMETADATA_H #include "llvm/Frontend/HLSL/HLSLRootSignature.h" +#include "llvm/IR/Constants.h" +#include "llvm/MC/DXContainerRootSignature.h" namespace llvm { class LLVMContext; @@ -49,6 +51,48 @@ private: SmallVector<Metadata *> GeneratedMetadata; }; +enum class RootSignatureElementKind { + Error = 0, + RootFlags = 1, + RootConstants = 2, + SRV = 3, + UAV = 4, + CBV = 5, + DescriptorTable = 6, + StaticSamplers = 7 +}; + +class MetadataParser { +public: + MetadataParser(MDNode *Root) : Root(Root) {} + + LLVM_ABI bool ParseRootSignature(LLVMContext *Ctx, + mcdxbc::RootSignatureDesc &RSD); + +private: + bool parseRootFlags(LLVMContext *Ctx, mcdxbc::RootSignatureDesc &RSD, + MDNode *RootFlagNode); + bool parseRootConstants(LLVMContext *Ctx, mcdxbc::RootSignatureDesc &RSD, + MDNode *RootConstantNode); + bool parseRootDescriptors(LLVMContext *Ctx, mcdxbc::RootSignatureDesc &RSD, + MDNode *RootDescriptorNode, + RootSignatureElementKind ElementKind); + bool parseDescriptorRange(LLVMContext *Ctx, mcdxbc::DescriptorTable &Table, + MDNode *RangeDescriptorNode); + bool parseDescriptorTable(LLVMContext *Ctx, mcdxbc::RootSignatureDesc &RSD, + MDNode *DescriptorTableNode); + bool parseRootSignatureElement(LLVMContext *Ctx, + mcdxbc::RootSignatureDesc &RSD, + MDNode *Element); + bool parseStaticSampler(LLVMContext *Ctx, mcdxbc::RootSignatureDesc &RSD, + MDNode *StaticSamplerNode); + + bool validateRootSignature(LLVMContext *Ctx, + const llvm::mcdxbc::RootSignatureDesc &RSD); + + MDNode *Root; +}; + } // namespace rootsig } // namespace hlsl } // namespace llvm diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h index de888ff..7919f7a 100644 --- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h +++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h @@ -779,16 +779,17 @@ struct LinkT { template <typename T, typename I, typename E> // struct MapT { using LocatorList = ObjectListT<I, E>; - ENUM(MapType, To, From, Tofrom, Alloc, Release, Delete); - ENUM(MapTypeModifier, Always, Close, Present, OmpxHold); + ENUM(MapType, To, From, Tofrom, Storage); + ENUM(MapTypeModifier, Always, Close, Delete, Present, Self, OmpxHold); + ENUM(RefModifier, RefPtee, RefPtr, RefPtrPtee); // See note at the definition of the MapperT type. using Mappers = ListT<type::MapperT<I, E>>; // Not a spec name using Iterator = type::IteratorT<T, I, E>; using MapTypeModifiers = ListT<MapTypeModifier>; // Not a spec name using TupleTrait = std::true_type; - std::tuple<OPT(MapType), OPT(MapTypeModifiers), OPT(Mappers), OPT(Iterator), - LocatorList> + std::tuple<OPT(MapType), OPT(MapTypeModifiers), OPT(RefModifier), + OPT(Mappers), OPT(Iterator), LocatorList> t; }; diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h index 611bfe3..047baa3 100644 --- a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h +++ b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h @@ -708,6 +708,7 @@ bool ConstructDecompositionT<C, H>::applyClause( tomp::clause::MapT<TypeTy, IdTy, ExprTy>{ {/*MapType=*/MapType::Tofrom, /*MapTypeModifier=*/std::nullopt, + /*RefModifier=*/std::nullopt, /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt, /*LocatorList=*/std::move(tofrom)}}); dirTarget->clauses.push_back(map); @@ -969,8 +970,8 @@ bool ConstructDecompositionT<C, H>::applyClause( llvm::omp::Clause::OMPC_map, tomp::clause::MapT<TypeTy, IdTy, ExprTy>{ {/*MapType=*/MapType::Tofrom, /*MapTypeModifier=*/std::nullopt, - /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt, - /*LocatorList=*/std::move(tofrom)}}); + /*RefModifier=*/std::nullopt, /*Mapper=*/std::nullopt, + /*Iterator=*/std::nullopt, /*LocatorList=*/std::move(tofrom)}}); dirTarget->clauses.push_back(map); applied = true; diff --git a/llvm/include/llvm/IR/CallingConv.h b/llvm/include/llvm/IR/CallingConv.h index d68491e..ef761eb 100644 --- a/llvm/include/llvm/IR/CallingConv.h +++ b/llvm/include/llvm/IR/CallingConv.h @@ -284,6 +284,9 @@ namespace CallingConv { RISCV_VLSCall_32768 = 122, RISCV_VLSCall_65536 = 123, + // Calling convention for AMDGPU whole wave functions. + AMDGPU_Gfx_WholeWave = 124, + /// The highest possible ID. Must be some 2^k - 1. MaxID = 1023 }; @@ -294,8 +297,13 @@ namespace CallingConv { /// directly or indirectly via a call-like instruction. constexpr bool isCallableCC(CallingConv::ID CC) { switch (CC) { + // Called with special intrinsics: + // llvm.amdgcn.cs.chain case CallingConv::AMDGPU_CS_Chain: case CallingConv::AMDGPU_CS_ChainPreserve: + // llvm.amdgcn.call.whole.wave + case CallingConv::AMDGPU_Gfx_WholeWave: + // Hardware entry points: case CallingConv::AMDGPU_CS: case CallingConv::AMDGPU_ES: case CallingConv::AMDGPU_GS: diff --git a/llvm/include/llvm/IR/DebugInfo.h b/llvm/include/llvm/IR/DebugInfo.h index f8241a3..c529a86 100644 --- a/llvm/include/llvm/IR/DebugInfo.h +++ b/llvm/include/llvm/IR/DebugInfo.h @@ -39,30 +39,26 @@ class DbgVariableRecord; class Instruction; class Module; -/// Finds dbg.declare intrinsics declaring local variables as living in the +/// Finds dbg.declare records declaring local variables as living in the /// memory that 'V' points to. -LLVM_ABI TinyPtrVector<DbgDeclareInst *> findDbgDeclares(Value *V); -/// As above, for DVRDeclares. LLVM_ABI TinyPtrVector<DbgVariableRecord *> findDVRDeclares(Value *V); /// As above, for DVRValues. LLVM_ABI TinyPtrVector<DbgVariableRecord *> findDVRValues(Value *V); -/// Finds the llvm.dbg.value intrinsics describing a value. -LLVM_ABI void findDbgValues( - SmallVectorImpl<DbgValueInst *> &DbgValues, Value *V, - SmallVectorImpl<DbgVariableRecord *> *DbgVariableRecords = nullptr); - -/// Finds the debug info intrinsics describing a value. -LLVM_ABI void findDbgUsers( - SmallVectorImpl<DbgVariableIntrinsic *> &DbgInsts, Value *V, - SmallVectorImpl<DbgVariableRecord *> *DbgVariableRecords = nullptr); +/// Finds the debug info records describing a value. +LLVM_ABI void +findDbgUsers(Value *V, + SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords); +/// Finds the dbg.values describing a value. +LLVM_ABI void +findDbgValues(Value *V, + SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords); /// Find subprogram that is enclosing this scope. LLVM_ABI DISubprogram *getDISubprogram(const MDNode *Scope); /// Produce a DebugLoc to use for each dbg.declare that is promoted to a /// dbg.value. -LLVM_ABI DebugLoc getDebugValueLoc(DbgVariableIntrinsic *DII); LLVM_ABI DebugLoc getDebugValueLoc(DbgVariableRecord *DVR); /// Strip debug info in the module if it exists. @@ -192,13 +188,6 @@ using AssignmentInstRange = /// Iterators invalidated by adding or removing DIAssignID metadata to/from any /// instruction (including by deleting or cloning instructions). LLVM_ABI AssignmentInstRange getAssignmentInsts(DIAssignID *ID); -/// Return a range of instructions (typically just one) that perform the -/// assignment that \p DAI encodes. -/// Iterators invalidated by adding or removing DIAssignID metadata to/from any -/// instruction (including by deleting or cloning instructions). -inline AssignmentInstRange getAssignmentInsts(const DbgAssignIntrinsic *DAI) { - return getAssignmentInsts(DAI->getAssignID()); -} inline AssignmentInstRange getAssignmentInsts(const DbgVariableRecord *DVR) { assert(DVR->isDbgAssign() && diff --git a/llvm/include/llvm/IR/DiagnosticInfo.h b/llvm/include/llvm/IR/DiagnosticInfo.h index 862be04..5f7225e 100644 --- a/llvm/include/llvm/IR/DiagnosticInfo.h +++ b/llvm/include/llvm/IR/DiagnosticInfo.h @@ -68,6 +68,7 @@ enum DiagnosticKind { DK_StackSize, DK_Linker, DK_Lowering, + DK_LegalizationFailure, DK_DebugMetadataVersion, DK_DebugMetadataInvalid, DK_Instrumentation, @@ -383,6 +384,30 @@ private: DiagnosticLocation Loc; }; +class LLVM_ABI DiagnosticInfoLegalizationFailure + : public DiagnosticInfoWithLocationBase { +private: + /// Message to be reported. + const Twine &MsgStr; + +public: + DiagnosticInfoLegalizationFailure(const Twine &MsgStr LLVM_LIFETIME_BOUND, + const Function &Fn, + const DiagnosticLocation &Loc, + DiagnosticSeverity Severity = DS_Error) + : DiagnosticInfoWithLocationBase(DK_LegalizationFailure, Severity, Fn, + Loc), + MsgStr(MsgStr) {} + + const Twine &getMsgStr() const { return MsgStr; } + + void print(DiagnosticPrinter &DP) const override; + + static bool classof(const DiagnosticInfo *DI) { + return DI->getKind() == DK_LegalizationFailure; + } +}; + class LLVM_ABI DiagnosticInfoGenericWithLoc : public DiagnosticInfoWithLocationBase { private: diff --git a/llvm/include/llvm/IR/GCStrategy.h b/llvm/include/llvm/IR/GCStrategy.h index 6b81355..44e46e4 100644 --- a/llvm/include/llvm/IR/GCStrategy.h +++ b/llvm/include/llvm/IR/GCStrategy.h @@ -47,6 +47,7 @@ #ifndef LLVM_IR_GCSTRATEGY_H #define LLVM_IR_GCSTRATEGY_H +#include "llvm/Support/Compiler.h" #include "llvm/Support/Registry.h" #include <optional> #include <string> @@ -81,7 +82,7 @@ protected: bool UsesMetadata = false; ///< If set, backend must emit metadata tables. public: - GCStrategy(); + LLVM_ABI GCStrategy(); virtual ~GCStrategy() = default; /// Return the name of the GC strategy. This is the value of the collector @@ -145,7 +146,7 @@ using GCRegistry = Registry<GCStrategy>; extern template class LLVM_TEMPLATE_ABI Registry<GCStrategy>; /// Lookup the GCStrategy object associated with the given gc name. -std::unique_ptr<GCStrategy> getGCStrategy(const StringRef Name); +LLVM_ABI std::unique_ptr<GCStrategy> getGCStrategy(const StringRef Name); } // end namespace llvm diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h index 7c600e7..6d3d864 100644 --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -2614,6 +2614,8 @@ public: return CreateShuffleVector(V, PoisonValue::get(V->getType()), Mask, Name); } + Value *CreateVectorInterleave(ArrayRef<Value *> Ops, const Twine &Name = ""); + Value *CreateExtractValue(Value *Agg, ArrayRef<unsigned> Idxs, const Twine &Name = "") { if (auto *V = Folder.FoldExtractValue(Agg, Idxs)) diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h index 1568052..48735b0 100644 --- a/llvm/include/llvm/IR/Intrinsics.h +++ b/llvm/include/llvm/IR/Intrinsics.h @@ -283,8 +283,15 @@ namespace Intrinsic { // or of the wrong kind will be renamed by adding ".renamed" to the name. LLVM_ABI std::optional<Function *> remangleIntrinsicFunction(Function *F); -} // End Intrinsic namespace + /// Returns the corresponding llvm.vector.interleaveN intrinsic for factor N. + LLVM_ABI Intrinsic::ID getInterleaveIntrinsicID(unsigned Factor); -} // End llvm namespace + /// Returns the corresponding llvm.vector.deinterleaveN intrinsic for factor + /// N. + LLVM_ABI Intrinsic::ID getDeinterleaveIntrinsicID(unsigned Factor); + + } // namespace Intrinsic + + } // namespace llvm #endif diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index ecda6c4..1da4e36 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +def flat_ptr_ty : LLVMQualPointerType<0>; def global_ptr_ty : LLVMQualPointerType<1>; def local_ptr_ty : LLVMQualPointerType<3>; @@ -3045,6 +3046,24 @@ def int_amdgcn_ds_bpermute_fi_b32 : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrConvergent, IntrWillReturn, IntrNoCallback, IntrNoFree]>; +def int_amdgcn_flat_prefetch : ClangBuiltin<"__builtin_amdgcn_flat_prefetch">, + Intrinsic<[], + [llvm_ptr_ty, // Pointer + llvm_i32_ty], // cachepolicy(imm), bits [0-2] = th, bits [3-4] = scope + [IntrInaccessibleMemOrArgMemOnly, IntrWillReturn, NoCapture<ArgIndex<0>>, + IntrNoCallback, IntrNoFree, ImmArg<ArgIndex<1>>], + "", [SDNPMemOperand] + >; + +def int_amdgcn_global_prefetch : ClangBuiltin<"__builtin_amdgcn_global_prefetch">, + Intrinsic<[], + [LLVMQualPointerType<1>, // Pointer + llvm_i32_ty], // cachepolicy(imm), bits [0-2] = th, bits [3-4] = scope + [IntrInaccessibleMemOrArgMemOnly, IntrWillReturn, NoCapture<ArgIndex<0>>, + IntrNoCallback, IntrNoFree, ImmArg<ArgIndex<1>>], + "", [SDNPMemOperand] + >; + //===----------------------------------------------------------------------===// // Deep learning intrinsics. //===----------------------------------------------------------------------===// @@ -3624,6 +3643,50 @@ def int_amdgcn_fdiv_fast : DefaultAttrsIntrinsic< [IntrNoMem, IntrSpeculatable] >; +class AMDGPUAsyncGlobalLoadToLDS : Intrinsic < + [], + [global_ptr_ty, // Base global pointer to load from + local_ptr_ty, // LDS base pointer to store to. + llvm_i32_ty, // offset + llvm_i32_ty], // gfx12+ cachepolicy: + // bits [0-2] = th + // bits [3-4] = scope + [IntrInaccessibleMemOrArgMemOnly, ReadOnly<ArgIndex<0>>, WriteOnly<ArgIndex<1>>, NoCapture<ArgIndex<0>>, + NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, IntrWillReturn, IntrNoCallback, IntrNoFree], + "", [SDNPMemOperand] +>; + +class AMDGPUAsyncGlobalStoreFromLDS : Intrinsic < + [], + [global_ptr_ty, // Base global pointer to store to + local_ptr_ty, // LDS base pointer to load from + llvm_i32_ty, // offset + llvm_i32_ty], // gfx12+ cachepolicy: + // bits [0-2] = th + // bits [3-4] = scope + [IntrInaccessibleMemOrArgMemOnly, WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>, NoCapture<ArgIndex<0>>, + NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, IntrWillReturn, IntrNoCallback, IntrNoFree], + "", [SDNPMemOperand] +>; + +def int_amdgcn_global_load_async_to_lds_b8 : + ClangBuiltin<"__builtin_amdgcn_global_load_async_to_lds_b8">, AMDGPUAsyncGlobalLoadToLDS; +def int_amdgcn_global_load_async_to_lds_b32 : + ClangBuiltin<"__builtin_amdgcn_global_load_async_to_lds_b32">, AMDGPUAsyncGlobalLoadToLDS; +def int_amdgcn_global_load_async_to_lds_b64 : + ClangBuiltin<"__builtin_amdgcn_global_load_async_to_lds_b64">, AMDGPUAsyncGlobalLoadToLDS; +def int_amdgcn_global_load_async_to_lds_b128 : + ClangBuiltin<"__builtin_amdgcn_global_load_async_to_lds_b128">, AMDGPUAsyncGlobalLoadToLDS; + +def int_amdgcn_global_store_async_from_lds_b8 : + ClangBuiltin<"__builtin_amdgcn_global_store_async_from_lds_b8">, AMDGPUAsyncGlobalStoreFromLDS; +def int_amdgcn_global_store_async_from_lds_b32 : + ClangBuiltin<"__builtin_amdgcn_global_store_async_from_lds_b32">, AMDGPUAsyncGlobalStoreFromLDS; +def int_amdgcn_global_store_async_from_lds_b64 : + ClangBuiltin<"__builtin_amdgcn_global_store_async_from_lds_b64">, AMDGPUAsyncGlobalStoreFromLDS; +def int_amdgcn_global_store_async_from_lds_b128 : + ClangBuiltin<"__builtin_amdgcn_global_store_async_from_lds_b128">, AMDGPUAsyncGlobalStoreFromLDS; + // WMMA intrinsics. class AMDGPUWmmaIntrinsicModsAB<LLVMType AB, LLVMType CD> : Intrinsic< @@ -3717,6 +3780,20 @@ class AMDGPUWmmaIntrinsicModsAllDiff<LLVMType DstTy, LLVMType AB, LLVMType C> : IntrWillReturn, IntrNoCallback, IntrNoFree] >; +class AMDGPUWmmaIntrinsicModsC_MatrixFMT : + Intrinsic< + [llvm_anyfloat_ty], // %D + [ + llvm_i32_ty, // matrix_a_fmt + llvm_anyint_ty, // %A + llvm_i32_ty, // matrix_b_fmt + llvm_anyint_ty, // %B + llvm_i16_ty, // %C_mod: 0 - none, 1 - neg, 2 - abs, 3 - neg(abs) + LLVMMatchType<0>, // %C + ], + [IntrNoMem, IntrConvergent, ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<4>>, IntrWillReturn, IntrNoCallback, IntrNoFree] +>; + defset list<Intrinsic> AMDGPUWMMAIntrinsicsGFX1250 = { def int_amdgcn_wmma_f32_16x16x4_f32 : AMDGPUWmmaIntrinsicModsAllReuse<llvm_anyfloat_ty, llvm_anyfloat_ty>; def int_amdgcn_wmma_f32_16x16x32_bf16 : AMDGPUWmmaIntrinsicModsAllReuse<llvm_anyfloat_ty, llvm_anyfloat_ty>; @@ -3741,6 +3818,7 @@ def int_amdgcn_wmma_f32_16x16x128_fp8_bf8 : AMDGPUWmmaIntrinsicModsC<llvm_anyint def int_amdgcn_wmma_f32_16x16x128_bf8_fp8 : AMDGPUWmmaIntrinsicModsC<llvm_anyint_ty, llvm_anyfloat_ty>; def int_amdgcn_wmma_f32_16x16x128_bf8_bf8 : AMDGPUWmmaIntrinsicModsC<llvm_anyint_ty, llvm_anyfloat_ty>; def int_amdgcn_wmma_i32_16x16x64_iu8 : AMDGPUWmmaIntrinsicModsAB<llvm_anyint_ty, llvm_anyint_ty>; +def int_amdgcn_wmma_f32_16x16x128_f8f6f4 : AMDGPUWmmaIntrinsicModsC_MatrixFMT; def int_amdgcn_wmma_f32_32x16x128_f4 : AMDGPUWmmaIntrinsicF4ModsC<llvm_anyint_ty, llvm_anyint_ty, llvm_anyfloat_ty>; } @@ -3813,6 +3891,26 @@ def int_amdgcn_tensor_load_to_lds_d2 : def int_amdgcn_tensor_store_from_lds_d2 : ClangBuiltin<"__builtin_amdgcn_tensor_store_from_lds_d2">, AMDGPUTensorLoadStoreD2; +class AMDGPULoadMonitor<LLVMType ptr_ty>: + Intrinsic< + [llvm_any_ty], + [ptr_ty, + llvm_i32_ty], // gfx12+ cachepolicy: + // bits [0-2] = th + // bits [3-4] = scope + [IntrArgMemOnly, IntrReadMem, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<1>>, + IntrWillReturn, IntrConvergent, IntrNoCallback, IntrNoFree], + "", + [SDNPMemOperand] + >; + +def int_amdgcn_flat_load_monitor_b32 : AMDGPULoadMonitor<flat_ptr_ty>; +def int_amdgcn_flat_load_monitor_b64 : AMDGPULoadMonitor<flat_ptr_ty>; +def int_amdgcn_flat_load_monitor_b128 : AMDGPULoadMonitor<flat_ptr_ty>; +def int_amdgcn_global_load_monitor_b32 : AMDGPULoadMonitor<global_ptr_ty>; +def int_amdgcn_global_load_monitor_b64 : AMDGPULoadMonitor<global_ptr_ty>; +def int_amdgcn_global_load_monitor_b128 : AMDGPULoadMonitor<global_ptr_ty>; + /// Emit an addrspacecast without null pointer checking. /// Should only be inserted by a pass based on analysis of an addrspacecast's src. def int_amdgcn_addrspacecast_nonnull : DefaultAttrsIntrinsic< diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td index 5ddc144..967d166 100644 --- a/llvm/include/llvm/IR/IntrinsicsNVVM.td +++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td @@ -331,6 +331,11 @@ class WMMA_REGS<string Geom, string Frag, string PtxEltType> { !eq(gf,"m8n16:x2") : !listsplat(llvm_i32_ty, 2), !eq(gf,"m8n16:x4") : !listsplat(llvm_i32_ty, 4), + // stmatrix b8 -> s32 @ m16n8 + !eq(gf,"m16n8:x1") : !listsplat(llvm_i32_ty, 1), + !eq(gf,"m16n8:x2") : !listsplat(llvm_i32_ty, 2), + !eq(gf,"m16n8:x4") : !listsplat(llvm_i32_ty, 4), + ); } @@ -403,6 +408,17 @@ class LDMATRIX_NAME<WMMA_REGS Frag, int Trans> { !subst("llvm.", "int_", intr)); } +class STMATRIX_NAME<WMMA_REGS Frag, int Trans> { + string intr = "llvm.nvvm.stmatrix.sync.aligned" + # "." # Frag.geom + # "." # Frag.frag + # !if(Trans, ".trans", "") + # "." # Frag.ptx_elt_type + ; + string record = !subst(".", "_", + !subst("llvm.", "int_", intr)); +} + // Generates list of 4-tuples of WMMA_REGS representing a valid MMA op. // Geom: list of supported geometries. // TypeN: PTX type of the corresponding fragment's element. @@ -443,6 +459,16 @@ class LDMATRIX_OPS<list<string> Geom, list<string> Frags, list<string> Types> { list<string> ops = !foreach(x, ret, x.gft); } +class STMATRIX_OPS<list<string> Geom, list<string> Frags, list<string> Types> { + list<WMMA_REGS> ret = + !foldl([]<WMMA_REGS>, Geom, t1, geom, !listconcat(t1, + !foldl([]<WMMA_REGS>, Frags, t2, frag, !listconcat(t2, + !foldl([]<WMMA_REGS>, Types, t3, type, !listconcat(t3, + [WMMA_REGS<geom, frag, type>])))))); + // Debugging aid for readable representation of the list above. + list<string> ops = !foreach(x, ret, x.gft); +} + // Creates list of valid combinations of fragments. This is the main list that // drives generation of corresponding intrinsics and instructions. class NVVM_MMA_OPS { @@ -537,9 +563,18 @@ class NVVM_MMA_OPS { list<WMMA_REGS> ldmatrix_geom_m8n16_ops = LDMATRIX_OPS< ["m8n16"], ["x1", "x2", "x4"], ["b8x16.b6x16_p32", "b8x16.b4x16_p64"]>.ret; + list<WMMA_REGS> stmatrix_b16_ops = STMATRIX_OPS< + ["m8n8"], ["x1", "x2", "x4"], ["b16"]>.ret; + + list<WMMA_REGS> stmatrix_b8_ops = STMATRIX_OPS< + ["m16n8"], ["x1", "x2", "x4"], ["b8"]>.ret; + list<WMMA_REGS> all_ldmatrix_ops = !listconcat(ldmatrix_b16_ops, ldmatrix_geom_m16n16_ops, ldmatrix_geom_m8n16_ops); + + list<WMMA_REGS> all_stmatrix_ops = !listconcat(stmatrix_b16_ops, + stmatrix_b8_ops); } def NVVM_MMA_OPS : NVVM_MMA_OPS; @@ -680,6 +715,19 @@ class NVVM_LDMATRIX_SUPPORTED<WMMA_REGS frag, bit trans> { ); } +// Returns true if the fragment is valid for stmatrix ops is supported; +// false otherwise. +class NVVM_STMATRIX_SUPPORTED<WMMA_REGS frag, bit trans> { + string g = frag.geom; + string t = frag.ptx_elt_type; + + bit ret = !cond( + !and(!eq(g, "m8n8"), !eq(t, "b16")): true, + !and(!eq(g, "m16n8"), !eq(t, "b8"), !eq(trans, 1)): true, + true: false + ); +} + class SHFL_INFO<bit sync, string mode, string type, bit return_pred> { string Suffix = !if(sync, "sync_", "") # mode # "_" @@ -1969,6 +2017,23 @@ foreach transposed = [0, 1] in { } } +// STMATRIX +class NVVM_STMATRIX<WMMA_REGS Frag, int Transposed> + : Intrinsic<[], + !listconcat([llvm_anyptr_ty], Frag.regs), + [IntrWriteMem, IntrArgMemOnly, IntrNoCallback, + WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], + STMATRIX_NAME<Frag, Transposed>.intr>; + +foreach transposed = [0, 1] in { + foreach frag = NVVM_MMA_OPS.all_stmatrix_ops in { + if NVVM_STMATRIX_SUPPORTED<frag, transposed>.ret then { + def STMATRIX_NAME<frag, transposed>.record + : NVVM_STMATRIX<frag, transposed>; + } + } +} + // MAPA let IntrProperties = [IntrNoMem, IntrSpeculatable, NoCapture<ArgIndex<0>>] in { def int_nvvm_mapa diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td index f592ff2..c1e4b97 100644 --- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td +++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td @@ -43,6 +43,10 @@ def int_wasm_ref_is_null_exn : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_exnref_ty], [IntrNoMem], "llvm.wasm.ref.is_null.exn">; +def int_wasm_ref_test_func + : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_vararg_ty], + [IntrNoMem]>; + //===----------------------------------------------------------------------===// // Table intrinsics //===----------------------------------------------------------------------===// diff --git a/llvm/include/llvm/IR/Metadata.h b/llvm/include/llvm/IR/Metadata.h index af252aa..33203ad 100644 --- a/llvm/include/llvm/IR/Metadata.h +++ b/llvm/include/llvm/IR/Metadata.h @@ -759,18 +759,18 @@ public: /// memory access used by the alias-analysis infrastructure. struct AAMDNodes { explicit AAMDNodes() = default; - explicit AAMDNodes(MDNode *T, MDNode *TS, MDNode *S, MDNode *N) - : TBAA(T), TBAAStruct(TS), Scope(S), NoAlias(N) {} + explicit AAMDNodes(MDNode *T, MDNode *TS, MDNode *S, MDNode *N, MDNode *NAS) + : TBAA(T), TBAAStruct(TS), Scope(S), NoAlias(N), NoAliasAddrSpace(NAS) {} bool operator==(const AAMDNodes &A) const { return TBAA == A.TBAA && TBAAStruct == A.TBAAStruct && Scope == A.Scope && - NoAlias == A.NoAlias; + NoAlias == A.NoAlias && NoAliasAddrSpace == A.NoAliasAddrSpace; } bool operator!=(const AAMDNodes &A) const { return !(*this == A); } explicit operator bool() const { - return TBAA || TBAAStruct || Scope || NoAlias; + return TBAA || TBAAStruct || Scope || NoAlias || NoAliasAddrSpace; } /// The tag for type-based alias analysis. @@ -785,6 +785,9 @@ struct AAMDNodes { /// The tag specifying the noalias scope. MDNode *NoAlias = nullptr; + /// The tag specifying the noalias address spaces. + MDNode *NoAliasAddrSpace = nullptr; + // Shift tbaa Metadata node to start off bytes later LLVM_ABI static MDNode *shiftTBAA(MDNode *M, size_t off); @@ -806,6 +809,8 @@ struct AAMDNodes { Result.TBAAStruct = Other.TBAAStruct == TBAAStruct ? TBAAStruct : nullptr; Result.Scope = Other.Scope == Scope ? Scope : nullptr; Result.NoAlias = Other.NoAlias == NoAlias ? NoAlias : nullptr; + Result.NoAliasAddrSpace = + Other.NoAliasAddrSpace == NoAliasAddrSpace ? NoAliasAddrSpace : nullptr; return Result; } @@ -818,6 +823,7 @@ struct AAMDNodes { TBAAStruct ? shiftTBAAStruct(TBAAStruct, Offset) : nullptr; Result.Scope = Scope; Result.NoAlias = NoAlias; + Result.NoAliasAddrSpace = NoAliasAddrSpace; return Result; } @@ -833,6 +839,7 @@ struct AAMDNodes { Result.TBAAStruct = TBAAStruct; Result.Scope = Scope; Result.NoAlias = NoAlias; + Result.NoAliasAddrSpace = NoAliasAddrSpace; return Result; } @@ -860,12 +867,12 @@ struct AAMDNodes { template<> struct DenseMapInfo<AAMDNodes> { static inline AAMDNodes getEmptyKey() { - return AAMDNodes(DenseMapInfo<MDNode *>::getEmptyKey(), - nullptr, nullptr, nullptr); + return AAMDNodes(DenseMapInfo<MDNode *>::getEmptyKey(), nullptr, nullptr, + nullptr, nullptr); } static inline AAMDNodes getTombstoneKey() { - return AAMDNodes(DenseMapInfo<MDNode *>::getTombstoneKey(), + return AAMDNodes(DenseMapInfo<MDNode *>::getTombstoneKey(), nullptr, nullptr, nullptr, nullptr); } @@ -873,7 +880,8 @@ struct DenseMapInfo<AAMDNodes> { return DenseMapInfo<MDNode *>::getHashValue(Val.TBAA) ^ DenseMapInfo<MDNode *>::getHashValue(Val.TBAAStruct) ^ DenseMapInfo<MDNode *>::getHashValue(Val.Scope) ^ - DenseMapInfo<MDNode *>::getHashValue(Val.NoAlias); + DenseMapInfo<MDNode *>::getHashValue(Val.NoAlias) ^ + DenseMapInfo<MDNode *>::getHashValue(Val.NoAliasAddrSpace); } static bool isEqual(const AAMDNodes &LHS, const AAMDNodes &RHS) { diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h index 737610b..11bfd73 100644 --- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h +++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h @@ -112,7 +112,6 @@ inline bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) { return false; } llvm_unreachable("Checking FTZ flag for invalid f2i/d2i intrinsic"); - return false; } inline bool FPToIntegerIntrinsicResultIsSigned(Intrinsic::ID IntrinsicID) { @@ -179,7 +178,6 @@ inline bool FPToIntegerIntrinsicResultIsSigned(Intrinsic::ID IntrinsicID) { } llvm_unreachable( "Checking invalid f2i/d2i intrinsic for signed int conversion"); - return false; } inline APFloat::roundingMode @@ -250,7 +248,6 @@ GetFPToIntegerRoundingMode(Intrinsic::ID IntrinsicID) { return APFloat::rmTowardZero; } llvm_unreachable("Checking rounding mode for invalid f2i/d2i intrinsic"); - return APFloat::roundingMode::Invalid; } inline bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID) { @@ -280,7 +277,6 @@ inline bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID) { return false; } llvm_unreachable("Checking FTZ flag for invalid fmin/fmax intrinsic"); - return false; } inline bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID) { @@ -310,7 +306,6 @@ inline bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID) { return false; } llvm_unreachable("Checking NaN flag for invalid fmin/fmax intrinsic"); - return false; } inline bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID) { @@ -340,7 +335,83 @@ inline bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID) { return false; } llvm_unreachable("Checking XorSignAbs flag for invalid fmin/fmax intrinsic"); - return false; +} + +inline bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) { + switch (IntrinsicID) { + case Intrinsic::nvvm_ceil_ftz_f: + case Intrinsic::nvvm_fabs_ftz: + case Intrinsic::nvvm_floor_ftz_f: + case Intrinsic::nvvm_round_ftz_f: + case Intrinsic::nvvm_saturate_ftz_f: + case Intrinsic::nvvm_sqrt_rn_ftz_f: + return true; + case Intrinsic::nvvm_ceil_f: + case Intrinsic::nvvm_ceil_d: + case Intrinsic::nvvm_fabs: + case Intrinsic::nvvm_floor_f: + case Intrinsic::nvvm_floor_d: + case Intrinsic::nvvm_round_f: + case Intrinsic::nvvm_round_d: + case Intrinsic::nvvm_saturate_d: + case Intrinsic::nvvm_saturate_f: + case Intrinsic::nvvm_sqrt_f: + case Intrinsic::nvvm_sqrt_rn_d: + case Intrinsic::nvvm_sqrt_rn_f: + return false; + } + llvm_unreachable("Checking FTZ flag for invalid unary intrinsic"); +} + +inline bool RCPShouldFTZ(Intrinsic::ID IntrinsicID) { + switch (IntrinsicID) { + case Intrinsic::nvvm_rcp_rm_ftz_f: + case Intrinsic::nvvm_rcp_rn_ftz_f: + case Intrinsic::nvvm_rcp_rp_ftz_f: + case Intrinsic::nvvm_rcp_rz_ftz_f: + return true; + case Intrinsic::nvvm_rcp_rm_d: + case Intrinsic::nvvm_rcp_rm_f: + case Intrinsic::nvvm_rcp_rn_d: + case Intrinsic::nvvm_rcp_rn_f: + case Intrinsic::nvvm_rcp_rp_d: + case Intrinsic::nvvm_rcp_rp_f: + case Intrinsic::nvvm_rcp_rz_d: + case Intrinsic::nvvm_rcp_rz_f: + return false; + } + llvm_unreachable("Checking FTZ flag for invalid rcp intrinsic"); +} + +inline APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID) { + switch (IntrinsicID) { + case Intrinsic::nvvm_rcp_rm_f: + case Intrinsic::nvvm_rcp_rm_d: + case Intrinsic::nvvm_rcp_rm_ftz_f: + return APFloat::rmTowardNegative; + + case Intrinsic::nvvm_rcp_rn_f: + case Intrinsic::nvvm_rcp_rn_d: + case Intrinsic::nvvm_rcp_rn_ftz_f: + return APFloat::rmNearestTiesToEven; + + case Intrinsic::nvvm_rcp_rp_f: + case Intrinsic::nvvm_rcp_rp_d: + case Intrinsic::nvvm_rcp_rp_ftz_f: + return APFloat::rmTowardPositive; + + case Intrinsic::nvvm_rcp_rz_f: + case Intrinsic::nvvm_rcp_rz_d: + case Intrinsic::nvvm_rcp_rz_ftz_f: + return APFloat::rmTowardZero; + } + llvm_unreachable("Checking rounding mode for invalid rcp intrinsic"); +} + +inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) { + if (ShouldFTZ) + return DenormalMode::getPreserveSign(); + return DenormalMode::getIEEE(); } } // namespace nvvm diff --git a/llvm/include/llvm/IR/PassInstrumentation.h b/llvm/include/llvm/IR/PassInstrumentation.h index 0315715..33eda5a 100644 --- a/llvm/include/llvm/IR/PassInstrumentation.h +++ b/llvm/include/llvm/IR/PassInstrumentation.h @@ -164,7 +164,7 @@ public: /// Add a class name to pass name mapping for use by pass instrumentation. LLVM_ABI void addClassToPassName(StringRef ClassName, StringRef PassName); - /// Get the pass name for a given pass class name. + /// Get the pass name for a given pass class name. Empty if no match found. LLVM_ABI StringRef getPassNameForClassName(StringRef ClassName); private: diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 50e50a9..27c5d5c 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -822,12 +822,52 @@ template <typename Class> struct bind_ty { } }; +/// Check whether the value has the given Class and matches the nested +/// pattern. Capture it into the provided variable if successful. +template <typename Class, typename MatchTy> struct bind_and_match_ty { + Class *&VR; + MatchTy Match; + + bind_and_match_ty(Class *&V, const MatchTy &Match) : VR(V), Match(Match) {} + + template <typename ITy> bool match(ITy *V) const { + auto *CV = dyn_cast<Class>(V); + if (CV && Match.match(V)) { + VR = CV; + return true; + } + return false; + } +}; + /// Match a value, capturing it if we match. inline bind_ty<Value> m_Value(Value *&V) { return V; } inline bind_ty<const Value> m_Value(const Value *&V) { return V; } +/// Match against the nested pattern, and capture the value if we match. +template <typename MatchTy> +inline bind_and_match_ty<Value, MatchTy> m_Value(Value *&V, + const MatchTy &Match) { + return {V, Match}; +} + +/// Match against the nested pattern, and capture the value if we match. +template <typename MatchTy> +inline bind_and_match_ty<const Value, MatchTy> m_Value(const Value *&V, + const MatchTy &Match) { + return {V, Match}; +} + /// Match an instruction, capturing it if we match. inline bind_ty<Instruction> m_Instruction(Instruction *&I) { return I; } + +/// Match against the nested pattern, and capture the instruction if we match. +template <typename MatchTy> +inline bind_and_match_ty<Instruction, MatchTy> +m_Instruction(Instruction *&I, const MatchTy &Match) { + return {I, Match}; +} + /// Match a unary operator, capturing it if we match. inline bind_ty<UnaryOperator> m_UnOp(UnaryOperator *&I) { return I; } /// Match a binary operator, capturing it if we match. diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.h b/llvm/include/llvm/IR/RuntimeLibcalls.h index 89ad4e5..eb882c4 100644 --- a/llvm/include/llvm/IR/RuntimeLibcalls.h +++ b/llvm/include/llvm/IR/RuntimeLibcalls.h @@ -134,7 +134,7 @@ struct RuntimeLibcallsInfo { /// Check if this is valid libcall for the current module, otherwise /// RTLIB::Unsupported. - RTLIB::LibcallImpl getSupportedLibcallImpl(StringRef FuncName) const; + LLVM_ABI RTLIB::LibcallImpl getSupportedLibcallImpl(StringRef FuncName) const; private: static const RTLIB::LibcallImpl diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 2e231cf..31801da 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -119,7 +119,6 @@ LLVM_ABI void initializeExpandVariadicsPass(PassRegistry &); LLVM_ABI void initializeExternalAAWrapperPassPass(PassRegistry &); LLVM_ABI void initializeFEntryInserterLegacyPass(PassRegistry &); LLVM_ABI void initializeFinalizeISelPass(PassRegistry &); -LLVM_ABI void initializeFinalizeMachineBundlesPass(PassRegistry &); LLVM_ABI void initializeFixIrreduciblePass(PassRegistry &); LLVM_ABI void initializeFixupStatepointCallerSavedLegacyPass(PassRegistry &); LLVM_ABI void initializeFlattenCFGLegacyPassPass(PassRegistry &); diff --git a/llvm/include/llvm/LinkAllIR.h b/llvm/include/llvm/LinkAllIR.h index ceed784..894a8dd 100644 --- a/llvm/include/llvm/LinkAllIR.h +++ b/llvm/include/llvm/LinkAllIR.h @@ -21,6 +21,7 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" +#include "llvm/Support/AlwaysTrue.h" #include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Memory.h" @@ -29,19 +30,16 @@ #include "llvm/Support/Process.h" #include "llvm/Support/Program.h" #include "llvm/Support/Signals.h" -#include <cstdlib> namespace { struct ForceVMCoreLinking { ForceVMCoreLinking() { // We must reference VMCore in such a way that compilers will not - // delete it all as dead code, even with whole program optimization, - // yet is effectively a NO-OP. As the compiler isn't smart enough - // to know that getenv() never returns -1, this will do the job. + // delete it all as dead code, even with whole program optimization. // This is so that globals in the translation units where these functions // are defined are forced to be initialized, populating various // registries. - if (std::getenv("bar") != (char*) -1) + if (llvm::getNonFoldableAlwaysTrue()) return; llvm::LLVMContext Context; (void)new llvm::Module("", Context); diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h index bae7f0d..f82a439 100644 --- a/llvm/include/llvm/LinkAllPasses.h +++ b/llvm/include/llvm/LinkAllPasses.h @@ -34,6 +34,7 @@ #include "llvm/CodeGen/Passes.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRPrintingPasses.h" +#include "llvm/Support/AlwaysTrue.h" #include "llvm/Support/Valgrind.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" @@ -54,14 +55,12 @@ class Triple; namespace { struct ForcePassLinking { ForcePassLinking() { - // We must reference the passes in such a way that compilers will not - // delete it all as dead code, even with whole program optimization, - // yet is effectively a NO-OP. As the compiler isn't smart enough - // to know that getenv() never returns -1, this will do the job. - // This is so that globals in the translation units where these functions - // are defined are forced to be initialized, populating various - // registries. - if (std::getenv("bar") != (char *)-1) + // We must reference the passes in such a way that compilers will not delete + // it all as dead code, even with whole program optimization, yet is + // effectively a NO-OP. This is so that globals in the translation units + // where these functions are defined are forced to be initialized, + // populating various registries. + if (llvm::getNonFoldableAlwaysTrue()) return; (void)llvm::createAtomicExpandLegacyPass(); diff --git a/llvm/include/llvm/MC/DXContainerRootSignature.h b/llvm/include/llvm/MC/DXContainerRootSignature.h index 4b6b42f..3c7c886 100644 --- a/llvm/include/llvm/MC/DXContainerRootSignature.h +++ b/llvm/include/llvm/MC/DXContainerRootSignature.h @@ -6,7 +6,11 @@ // //===----------------------------------------------------------------------===// +#ifndef LLVM_MC_DXCONTAINERROOTSIGNATURE_H +#define LLVM_MC_DXCONTAINERROOTSIGNATURE_H + #include "llvm/BinaryFormat/DXContainer.h" +#include "llvm/Support/Compiler.h" #include <cstdint> #include <limits> @@ -110,9 +114,11 @@ struct RootSignatureDesc { mcdxbc::RootParametersContainer ParametersContainer; SmallVector<dxbc::RTS0::v1::StaticSampler> StaticSamplers; - void write(raw_ostream &OS) const; + LLVM_ABI void write(raw_ostream &OS) const; - size_t getSize() const; + LLVM_ABI size_t getSize() const; }; } // namespace mcdxbc } // namespace llvm + +#endif // LLVM_MC_DXCONTAINERROOTSIGNATURE_H diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h index 0322cbe..bfc1175 100644 --- a/llvm/include/llvm/MC/MCAsmBackend.h +++ b/llvm/include/llvm/MC/MCAsmBackend.h @@ -18,9 +18,7 @@ namespace llvm { -class MCAlignFragment; class MCFragment; -class MCLEBFragment; class MCSymbol; class MCAssembler; class MCContext; @@ -60,6 +58,9 @@ protected: // Can only create subclasses. MCAssembler *Asm = nullptr; + bool AllowAutoPadding = false; + bool AllowEnhancedRelaxation = false; + public: MCAsmBackend(const MCAsmBackend &) = delete; MCAsmBackend &operator=(const MCAsmBackend &) = delete; @@ -73,11 +74,11 @@ public: /// Return true if this target might automatically pad instructions and thus /// need to emit padding enable/disable directives around sensative code. - virtual bool allowAutoPadding() const { return false; } + bool allowAutoPadding() const { return AllowAutoPadding; } /// Return true if this target allows an unrelaxable instruction to be /// emitted into RelaxableFragment and then we can increase its size in a /// tricky way for optimization. - virtual bool allowEnhancedRelaxation() const { return false; } + bool allowEnhancedRelaxation() const { return AllowEnhancedRelaxation; } /// lifetime management virtual void reset() {} @@ -105,21 +106,6 @@ public: /// Get information on a fixup kind. virtual MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const; - /// Hook to check if extra nop bytes must be inserted for alignment directive. - /// For some targets this may be necessary in order to support linker - /// relaxation. The number of bytes to insert are returned in Size. - virtual bool shouldInsertExtraNopBytesForCodeAlign(const MCAlignFragment &AF, - unsigned &Size) { - return false; - } - - /// Hook which indicates if the target requires a fixup to be generated when - /// handling an align directive in an executable section - virtual bool shouldInsertFixupForCodeAlign(MCAssembler &Asm, - MCAlignFragment &AF) { - return false; - } - // Evaluate a fixup, returning std::nullopt to use default handling for // `Value` and `IsResolved`. Otherwise, returns `IsResolved` with the // expectation that the hook updates `Value`. @@ -177,6 +163,10 @@ public: } // Defined by linker relaxation targets. + + // Return false to use default handling. Otherwise, set `Size` to the number + // of padding bytes. + virtual bool relaxAlign(MCFragment &F, unsigned &Size) { return false; } virtual bool relaxDwarfLineAddr(MCFragment &, bool &WasRelaxed) const { return false; } diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h index 71da048..6c12cd3 100644 --- a/llvm/include/llvm/MC/MCAsmInfo.h +++ b/llvm/include/llvm/MC/MCAsmInfo.h @@ -35,6 +35,7 @@ class MCStreamer; class MCSubtargetInfo; class MCSymbol; class MCValue; +class Triple; class raw_ostream; namespace WinEH { @@ -485,6 +486,9 @@ public: /// syntactically correct. virtual bool isValidUnquotedName(StringRef Name) const; + virtual void printSwitchToSection(const MCSection &, uint32_t Subsection, + const Triple &, raw_ostream &) const {} + /// Return true if the .section directive should be omitted when /// emitting \p SectionName. For example: /// @@ -494,6 +498,10 @@ public: /// returns true => .text virtual bool shouldOmitSectionDirective(StringRef SectionName) const; + // Return true if a .align directive should use "optimized nops" to fill + // instead of 0s. + virtual bool useCodeAlign(const MCSection &Sec) const { return false; } + bool usesSunStyleELFSectionSwitchSyntax() const { return SunStyleELFSectionSwitchSyntax; } diff --git a/llvm/include/llvm/MC/MCAsmInfoCOFF.h b/llvm/include/llvm/MC/MCAsmInfoCOFF.h index 1dfb475..dc7832c 100644 --- a/llvm/include/llvm/MC/MCAsmInfoCOFF.h +++ b/llvm/include/llvm/MC/MCAsmInfoCOFF.h @@ -15,6 +15,9 @@ namespace llvm { class MCAsmInfoCOFF : public MCAsmInfo { virtual void anchor(); + void printSwitchToSection(const MCSection &, uint32_t, const Triple &, + raw_ostream &) const final; + bool useCodeAlign(const MCSection &Sec) const final; protected: explicit MCAsmInfoCOFF(); diff --git a/llvm/include/llvm/MC/MCAsmInfoDarwin.h b/llvm/include/llvm/MC/MCAsmInfoDarwin.h index 4ca62b3..12bc3e9 100644 --- a/llvm/include/llvm/MC/MCAsmInfoDarwin.h +++ b/llvm/include/llvm/MC/MCAsmInfoDarwin.h @@ -21,6 +21,9 @@ namespace llvm { class MCAsmInfoDarwin : public MCAsmInfo { public: explicit MCAsmInfoDarwin(); + void printSwitchToSection(const MCSection &, uint32_t, const Triple &, + raw_ostream &) const final; + bool useCodeAlign(const MCSection &Sec) const final; /// True if the section is atomized using the symbols in it. /// This is false if the section is atomized based on its contents (MachO' __TEXT,__cstring for diff --git a/llvm/include/llvm/MC/MCAsmInfoELF.h b/llvm/include/llvm/MC/MCAsmInfoELF.h index 408d4df..c05e4ad 100644 --- a/llvm/include/llvm/MC/MCAsmInfoELF.h +++ b/llvm/include/llvm/MC/MCAsmInfoELF.h @@ -16,6 +16,9 @@ namespace llvm { class MCAsmInfoELF : public MCAsmInfo { virtual void anchor(); MCSection *getNonexecutableStackSection(MCContext &Ctx) const final; + void printSwitchToSection(const MCSection &, uint32_t, const Triple &, + raw_ostream &) const final; + bool useCodeAlign(const MCSection &Sec) const final; protected: MCAsmInfoELF(); diff --git a/llvm/include/llvm/MC/MCAsmInfoGOFF.h b/llvm/include/llvm/MC/MCAsmInfoGOFF.h index 1f3b263..e62d2ae 100644 --- a/llvm/include/llvm/MC/MCAsmInfoGOFF.h +++ b/llvm/include/llvm/MC/MCAsmInfoGOFF.h @@ -19,7 +19,8 @@ namespace llvm { class MCAsmInfoGOFF : public MCAsmInfo { - virtual void anchor(); + void printSwitchToSection(const MCSection &, uint32_t, const Triple &, + raw_ostream &) const final; protected: MCAsmInfoGOFF(); diff --git a/llvm/include/llvm/MC/MCAsmInfoWasm.h b/llvm/include/llvm/MC/MCAsmInfoWasm.h index 3afc610..d98de6c 100644 --- a/llvm/include/llvm/MC/MCAsmInfoWasm.h +++ b/llvm/include/llvm/MC/MCAsmInfoWasm.h @@ -13,7 +13,8 @@ namespace llvm { class MCAsmInfoWasm : public MCAsmInfo { - virtual void anchor(); + void printSwitchToSection(const MCSection &, uint32_t, const Triple &, + raw_ostream &) const final; protected: MCAsmInfoWasm(); diff --git a/llvm/include/llvm/MC/MCAsmInfoXCOFF.h b/llvm/include/llvm/MC/MCAsmInfoXCOFF.h index 5483899..fd1ae82 100644 --- a/llvm/include/llvm/MC/MCAsmInfoXCOFF.h +++ b/llvm/include/llvm/MC/MCAsmInfoXCOFF.h @@ -14,10 +14,11 @@ namespace llvm { class MCAsmInfoXCOFF : public MCAsmInfo { - virtual void anchor(); - protected: MCAsmInfoXCOFF(); + void printSwitchToSection(const MCSection &, uint32_t, const Triple &, + raw_ostream &) const final; + bool useCodeAlign(const MCSection &Sec) const final; public: // Return true only when C is an acceptable character inside a diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h index 467ad4e..4853701 100644 --- a/llvm/include/llvm/MC/MCAssembler.h +++ b/llvm/include/llvm/MC/MCAssembler.h @@ -209,7 +209,7 @@ public: LLVM_ABI bool registerSection(MCSection &Section); LLVM_ABI bool registerSymbol(const MCSymbol &Symbol); - void addRelocDirective(RelocDirective RD); + LLVM_ABI void addRelocDirective(RelocDirective RD); LLVM_ABI void reportError(SMLoc L, const Twine &Msg) const; // Record pending errors during layout iteration, as they may go away once the diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h index c137f61..ddac161 100644 --- a/llvm/include/llvm/MC/MCContext.h +++ b/llvm/include/llvm/MC/MCContext.h @@ -333,8 +333,6 @@ private: void reportCommon(SMLoc Loc, std::function<void(SMDiagnostic &, const SourceMgr *)>); - MCFragment *allocInitialFragment(MCSection &Sec); - MCSymbolTableEntry &getSymbolTableEntry(StringRef Name); MCSymbol *createSymbolImpl(const MCSymbolTableEntry *Name, bool IsTemporary); diff --git a/llvm/include/llvm/MC/MCMachObjectWriter.h b/llvm/include/llvm/MC/MCMachObjectWriter.h index 51e4df5..170e2e7 100644 --- a/llvm/include/llvm/MC/MCMachObjectWriter.h +++ b/llvm/include/llvm/MC/MCMachObjectWriter.h @@ -16,7 +16,7 @@ #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCLinkerOptimizationHint.h" #include "llvm/MC/MCObjectWriter.h" -#include "llvm/MC/MCSection.h" +#include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/StringTableBuilder.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/EndianStream.h" @@ -276,7 +276,7 @@ public: uint64_t SectionDataSize, uint32_t MaxProt, uint32_t InitProt); - void writeSection(const MCAssembler &Asm, const MCSection &Sec, + void writeSection(const MCAssembler &Asm, const MCSectionMachO &Sec, uint64_t VMAddr, uint64_t FileOffset, unsigned Flags, uint64_t RelocationsStart, unsigned NumRelocations); diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h index 319e131..5ac7aba 100644 --- a/llvm/include/llvm/MC/MCObjectStreamer.h +++ b/llvm/include/llvm/MC/MCObjectStreamer.h @@ -40,6 +40,7 @@ class MCObjectStreamer : public MCStreamer { std::unique_ptr<MCAssembler> Assembler; bool EmitEHFrame; bool EmitDebugFrame; + bool EmitSFrame; struct PendingAssignment { MCSymbol *Symbol; @@ -51,10 +52,13 @@ class MCObjectStreamer : public MCStreamer { DenseMap<const MCSymbol *, SmallVector<PendingAssignment, 1>> pendingAssignments; + SmallVector<std::unique_ptr<char[]>, 0> FragStorage; + // Available bytes in the current block for trailing data or new fragments. + size_t FragSpace = 0; + void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &); void emitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override; void emitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override; - void emitInstructionImpl(const MCInst &Inst, const MCSubtargetInfo &STI); protected: MCObjectStreamer(MCContext &Context, std::unique_ptr<MCAsmBackend> TAB, @@ -71,14 +75,7 @@ public: void emitFrames(MCAsmBackend *MAB); MCSymbol *emitCFILabel() override; - void emitCFISections(bool EH, bool Debug) override; - - /// Get a data fragment to write into, creating a new one if the current - /// fragment is not FT_Data. - MCFragment *getOrCreateDataFragment(); - -protected: - bool changeSectionImpl(MCSection *Section, uint32_t Subsection); + void emitCFISections(bool EH, bool Debug, bool SFrame) override; public: void visitUsedSymbol(const MCSymbol &Sym) override; @@ -88,6 +85,23 @@ public: /// \name MCStreamer Interface /// @{ + // Add a fragment with a variable-size tail and start a new empty fragment. + void insert(MCFragment *F); + + char *getCurFragEnd() const { + return reinterpret_cast<char *>(CurFrag + 1) + CurFrag->getFixedSize(); + } + MCFragment *allocFragSpace(size_t Headroom); + // Add a new fragment to the current section without a variable-size tail. + void newFragment(); + + void ensureHeadroom(size_t Headroom); + void appendContents(ArrayRef<char> Contents); + void appendContents(size_t Num, char Elt); + // Add a fixup to the current fragment. Call ensureHeadroom beforehand to + // ensure the fixup and appended content apply to the same fragment. + void addFixup(const MCExpr *Value, MCFixupKind Kind); + void emitLabel(MCSymbol *Symbol, SMLoc Loc = SMLoc()) override; virtual void emitLabelAtPos(MCSymbol *Symbol, SMLoc Loc, MCFragment &F, uint64_t Offset); @@ -100,7 +114,6 @@ public: void emitSLEB128Value(const MCExpr *Value) override; void emitWeakReference(MCSymbol *Alias, const MCSymbol *Target) override; void changeSection(MCSection *Section, uint32_t Subsection = 0) override; - void switchSectionNoPrint(MCSection *Section) override; void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override; /// Emit an instruction to a special fragment, because this instruction diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h index 66ea8f8..2e929d8 100644 --- a/llvm/include/llvm/MC/MCSection.h +++ b/llvm/include/llvm/MC/MCSection.h @@ -39,150 +39,6 @@ class MCSubtargetInfo; class raw_ostream; class Triple; -/// Instances of this class represent a uniqued identifier for a section in the -/// current translation unit. The MCContext class uniques and creates these. -class LLVM_ABI MCSection { -public: - friend MCAssembler; - friend MCObjectStreamer; - friend class MCFragment; - static constexpr unsigned NonUniqueID = ~0U; - - enum SectionVariant { - SV_COFF = 0, - SV_ELF, - SV_GOFF, - SV_MachO, - SV_Wasm, - SV_XCOFF, - SV_SPIRV, - SV_DXContainer, - }; - - struct iterator { - MCFragment *F = nullptr; - iterator() = default; - explicit iterator(MCFragment *F) : F(F) {} - MCFragment &operator*() const { return *F; } - bool operator==(const iterator &O) const { return F == O.F; } - bool operator!=(const iterator &O) const { return F != O.F; } - iterator &operator++(); - }; - - struct FragList { - MCFragment *Head = nullptr; - MCFragment *Tail = nullptr; - }; - -private: - // At parse time, this holds the fragment list of the current subsection. At - // layout time, this holds the concatenated fragment lists of all subsections. - FragList *CurFragList; - MCSymbol *Begin; - MCSymbol *End = nullptr; - /// The alignment requirement of this section. - Align Alignment; - /// The section index in the assemblers section list. - unsigned Ordinal = 0; - - /// Whether this section has had instructions emitted into it. - bool HasInstructions : 1; - - bool IsRegistered : 1; - - bool IsText : 1; - - bool IsVirtual : 1; - - /// Whether the section contains linker-relaxable fragments. If true, the - /// offset between two locations may not be fully resolved. - bool LinkerRelaxable : 1; - - // Mapping from subsection number to fragment list. At layout time, the - // subsection 0 list is replaced with concatenated fragments from all - // subsections. - SmallVector<std::pair<unsigned, FragList>, 1> Subsections; - - // Content and fixup storage for fragments - SmallVector<char, 0> ContentStorage; - SmallVector<MCFixup, 0> FixupStorage; - SmallVector<MCOperand, 0> MCOperandStorage; - -protected: - // TODO Make Name private when possible. - StringRef Name; - SectionVariant Variant; - - MCSection(SectionVariant V, StringRef Name, bool IsText, bool IsVirtual, - MCSymbol *Begin); - // Protected non-virtual dtor prevents destroy through a base class pointer. - ~MCSection() {} - -public: - MCSection(const MCSection &) = delete; - MCSection &operator=(const MCSection &) = delete; - - StringRef getName() const { return Name; } - bool isText() const { return IsText; } - - SectionVariant getVariant() const { return Variant; } - - MCSymbol *getBeginSymbol() { return Begin; } - const MCSymbol *getBeginSymbol() const { - return const_cast<MCSection *>(this)->getBeginSymbol(); - } - void setBeginSymbol(MCSymbol *Sym) { - assert(!Begin); - Begin = Sym; - } - MCSymbol *getEndSymbol(MCContext &Ctx); - bool hasEnded() const; - - Align getAlign() const { return Alignment; } - void setAlignment(Align Value) { Alignment = Value; } - - /// Makes sure that Alignment is at least MinAlignment. - void ensureMinAlignment(Align MinAlignment) { - if (Alignment < MinAlignment) - Alignment = MinAlignment; - } - - unsigned getOrdinal() const { return Ordinal; } - void setOrdinal(unsigned Value) { Ordinal = Value; } - - bool hasInstructions() const { return HasInstructions; } - void setHasInstructions(bool Value) { HasInstructions = Value; } - - bool isRegistered() const { return IsRegistered; } - void setIsRegistered(bool Value) { IsRegistered = Value; } - - bool isLinkerRelaxable() const { return LinkerRelaxable; } - void setLinkerRelaxable() { LinkerRelaxable = true; } - - MCFragment &getDummyFragment() { return *Subsections[0].second.Head; } - - FragList *curFragList() const { return CurFragList; } - iterator begin() const { return iterator(CurFragList->Head); } - iterator end() const { return {}; } - - void dump(DenseMap<const MCFragment *, SmallVector<const MCSymbol *, 0>> - *FragToSyms = nullptr) const; - - virtual void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, - raw_ostream &OS, - uint32_t Subsection) const = 0; - - /// Return true if a .align directive should use "optimized nops" to fill - /// instead of 0s. - virtual bool useCodeAlign() const = 0; - - /// Check whether this section is "virtual", that is has no actual object - /// file contents. - bool isVirtualSection() const { return IsVirtual; } - - virtual StringRef getVirtualSectionKind() const; -}; - // Represents a contiguous piece of code or data within a section. Its size is // determined by MCAssembler::layout. All subclasses must have trivial // destructors. @@ -234,11 +90,15 @@ protected: /// FT_Relaxable, x86-specific bool AllowAutoPadding : 1; - uint32_t ContentStart = 0; - uint32_t ContentEnd = 0; + // Track content and fixups for the fixed-size part as fragments are + // appended to the section. The content remains immutable, except when + // modified by applyFixup. + uint32_t FixedSize = 0; uint32_t FixupStart = 0; uint32_t FixupEnd = 0; + // Track content and fixups for the optional variable-size tail part, + // typically modified during relaxation. uint32_t VarContentStart = 0; uint32_t VarContentEnd = 0; uint32_t VarFixupStart = 0; @@ -255,6 +115,19 @@ protected: uint32_t OperandSize; } relax; struct { + // The alignment to ensure, in bytes. + Align Alignment; + // The size of the integer (in bytes) of \p Value. + uint8_t FillLen; + // If true, fill with target-specific nop instructions. + bool EmitNops; + // The maximum number of bytes to emit; if the alignment + // cannot be satisfied in this width then this fragment is ignored. + unsigned MaxBytesToEmit; + // Value to use for filling padding bytes. + int64_t Fill; + } align; + struct { // True if this is a sleb128, false if uleb128. bool IsSigned; // The value this fragment should contain. @@ -276,22 +149,6 @@ public: MCFragment(const MCFragment &) = delete; MCFragment &operator=(const MCFragment &) = delete; - bool isEncoded() const { - MCFragment::FragmentType Kind = getKind(); - switch (Kind) { - default: - return false; - case MCFragment::FT_Relaxable: - case MCFragment::FT_Data: - case MCFragment::FT_Dwarf: - case MCFragment::FT_DwarfFrame: - case MCFragment::FT_LEB: - case MCFragment::FT_CVInlineLines: - case MCFragment::FT_CVDefRange: - return true; - } - } - MCFragment *getNext() const { return Next; } FragmentType getKind() const { return Kind; } @@ -327,57 +184,21 @@ public: bool getAllowAutoPadding() const { return AllowAutoPadding; } void setAllowAutoPadding(bool V) { AllowAutoPadding = V; } - // Content-related functions manage parent's storage using ContentStart and + //== Content-related functions manage parent's storage using ContentStart and // ContentSize. - void clearContents() { ContentEnd = ContentStart; } - // Get a SmallVector reference. The caller should call doneAppending to update - // `ContentEnd`. - SmallVectorImpl<char> &getContentsForAppending() { - SmallVectorImpl<char> &S = getParent()->ContentStorage; - if (LLVM_UNLIKELY(ContentEnd != S.size())) { - // Move the elements to the end. Reserve space to avoid invalidating - // S.begin()+I for `append`. - auto Size = ContentEnd - ContentStart; - auto I = std::exchange(ContentStart, S.size()); - S.reserve(S.size() + Size); - S.append(S.begin() + I, S.begin() + I + Size); - } - return S; - } - void doneAppending() { ContentEnd = getParent()->ContentStorage.size(); } - void appendContents(ArrayRef<char> Contents) { - getContentsForAppending().append(Contents.begin(), Contents.end()); - doneAppending(); - } - void appendContents(size_t Num, char Elt) { - getContentsForAppending().append(Num, Elt); - doneAppending(); - } - LLVM_ABI void setContents(ArrayRef<char> Contents); - MutableArrayRef<char> getContents() { - return MutableArrayRef(getParent()->ContentStorage) - .slice(ContentStart, ContentEnd - ContentStart); - } - ArrayRef<char> getContents() const { - return ArrayRef(getParent()->ContentStorage) - .slice(ContentStart, ContentEnd - ContentStart); - } - void setVarContents(ArrayRef<char> Contents); + MutableArrayRef<char> getContents(); + ArrayRef<char> getContents() const; + + LLVM_ABI void setVarContents(ArrayRef<char> Contents); void clearVarContents() { setVarContents({}); } - MutableArrayRef<char> getVarContents() { - return MutableArrayRef(getParent()->ContentStorage) - .slice(VarContentStart, VarContentEnd - VarContentStart); - } - ArrayRef<char> getVarContents() const { - return ArrayRef(getParent()->ContentStorage) - .slice(VarContentStart, VarContentEnd - VarContentStart); - } + MutableArrayRef<char> getVarContents(); + ArrayRef<char> getVarContents() const; - size_t getFixedSize() const { return ContentEnd - ContentStart; } + size_t getFixedSize() const { return FixedSize; } size_t getVarSize() const { return VarContentEnd - VarContentStart; } size_t getSize() const { - return ContentEnd - ContentStart + (VarContentEnd - VarContentStart); + return FixedSize + (VarContentEnd - VarContentStart); } //== Fixup-related functions manage parent's storage using FixupStart and @@ -385,59 +206,55 @@ public: void clearFixups() { FixupEnd = FixupStart; } LLVM_ABI void addFixup(MCFixup Fixup); LLVM_ABI void appendFixups(ArrayRef<MCFixup> Fixups); - LLVM_ABI void setFixups(ArrayRef<MCFixup> Fixups); - MutableArrayRef<MCFixup> getFixups() { - return MutableArrayRef(getParent()->FixupStorage) - .slice(FixupStart, FixupEnd - FixupStart); - } - ArrayRef<MCFixup> getFixups() const { - return ArrayRef(getParent()->FixupStorage) - .slice(FixupStart, FixupEnd - FixupStart); - } + MutableArrayRef<MCFixup> getFixups(); + ArrayRef<MCFixup> getFixups() const; // Source fixup offsets are relative to the variable part's start. // Stored fixup offsets are relative to the fixed part's start. - void setVarFixups(ArrayRef<MCFixup> Fixups); + LLVM_ABI void setVarFixups(ArrayRef<MCFixup> Fixups); void clearVarFixups() { setVarFixups({}); } - MutableArrayRef<MCFixup> getVarFixups() { - return MutableArrayRef(getParent()->FixupStorage) - .slice(VarFixupStart, VarFixupEnd - VarFixupStart); - } - ArrayRef<MCFixup> getVarFixups() const { - return ArrayRef(getParent()->FixupStorage) - .slice(VarFixupStart, VarFixupEnd - VarFixupStart); - } + MutableArrayRef<MCFixup> getVarFixups(); + ArrayRef<MCFixup> getVarFixups() const; //== FT_Relaxable functions unsigned getOpcode() const { assert(Kind == FT_Relaxable); return u.relax.Opcode; } - ArrayRef<MCOperand> getOperands() const { - assert(Kind == FT_Relaxable); - return MutableArrayRef(getParent()->MCOperandStorage) - .slice(u.relax.OperandStart, u.relax.OperandSize); + ArrayRef<MCOperand> getOperands() const; + MCInst getInst() const; + void setInst(const MCInst &Inst); + + //== FT_Align functions + void makeAlign(Align Alignment, int64_t Fill, uint8_t FillLen, + unsigned MaxBytesToEmit) { + Kind = FT_Align; + u.align.EmitNops = false; + u.align.Alignment = Alignment; + u.align.Fill = Fill; + u.align.FillLen = FillLen; + u.align.MaxBytesToEmit = MaxBytesToEmit; } - MCInst getInst() const { - assert(Kind == FT_Relaxable); - MCInst Inst; - Inst.setOpcode(u.relax.Opcode); - Inst.setFlags(u.relax.Flags); - Inst.setOperands(ArrayRef(getParent()->MCOperandStorage) - .slice(u.relax.OperandStart, u.relax.OperandSize)); - return Inst; + + Align getAlignment() const { + assert(Kind == FT_Align); + return u.align.Alignment; } - void setInst(const MCInst &Inst) { - assert(Kind == FT_Relaxable); - u.relax.Opcode = Inst.getOpcode(); - u.relax.Flags = Inst.getFlags(); - auto &S = getParent()->MCOperandStorage; - if (Inst.getNumOperands() > u.relax.OperandSize) { - u.relax.OperandStart = S.size(); - S.resize_for_overwrite(S.size() + Inst.getNumOperands()); - } - u.relax.OperandSize = Inst.getNumOperands(); - llvm::copy(Inst, S.begin() + u.relax.OperandStart); + int64_t getAlignFill() const { + assert(Kind == FT_Align); + return u.align.Fill; + } + uint8_t getAlignFillLen() const { + assert(Kind == FT_Align); + return u.align.FillLen; + } + unsigned getAlignMaxBytesToEmit() const { + assert(Kind == FT_Align); + return u.align.MaxBytesToEmit; + } + bool hasAlignEmitNops() const { + assert(Kind == FT_Align); + return u.align.EmitNops; } //== FT_LEB functions @@ -487,52 +304,6 @@ protected: : MCFragment(FType, HasInstructions) {} }; -class MCAlignFragment : public MCFragment { - /// Flag to indicate that (optimal) NOPs should be emitted instead - /// of using the provided value. The exact interpretation of this flag is - /// target dependent. - bool EmitNops : 1; - - /// The alignment to ensure, in bytes. - Align Alignment; - - /// The size of the integer (in bytes) of \p Value. - uint8_t FillLen; - - /// The maximum number of bytes to emit; if the alignment - /// cannot be satisfied in this width then this fragment is ignored. - unsigned MaxBytesToEmit; - - /// Value to use for filling padding bytes. - int64_t Fill; - - /// When emitting Nops some subtargets have specific nop encodings. - const MCSubtargetInfo *STI = nullptr; - -public: - MCAlignFragment(Align Alignment, int64_t Fill, uint8_t FillLen, - unsigned MaxBytesToEmit) - : MCFragment(FT_Align, false), EmitNops(false), Alignment(Alignment), - FillLen(FillLen), MaxBytesToEmit(MaxBytesToEmit), Fill(Fill) {} - - Align getAlignment() const { return Alignment; } - int64_t getFill() const { return Fill; } - uint8_t getFillLen() const { return FillLen; } - unsigned getMaxBytesToEmit() const { return MaxBytesToEmit; } - - bool hasEmitNops() const { return EmitNops; } - void setEmitNops(bool Value, const MCSubtargetInfo *STI) { - EmitNops = Value; - this->STI = STI; - } - - const MCSubtargetInfo *getSubtargetInfo() const { return STI; } - - static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_Align; - } -}; - class MCFillFragment : public MCFragment { uint8_t ValueSize; /// Value to use for filling bytes. @@ -730,6 +501,190 @@ public: } }; +/// Instances of this class represent a uniqued identifier for a section in the +/// current translation unit. The MCContext class uniques and creates these. +class LLVM_ABI MCSection { +public: + friend MCAssembler; + friend MCObjectStreamer; + friend class MCFragment; + static constexpr unsigned NonUniqueID = ~0U; + + struct iterator { + MCFragment *F = nullptr; + iterator() = default; + explicit iterator(MCFragment *F) : F(F) {} + MCFragment &operator*() const { return *F; } + bool operator==(const iterator &O) const { return F == O.F; } + bool operator!=(const iterator &O) const { return F != O.F; } + iterator &operator++(); + }; + + struct FragList { + MCFragment *Head = nullptr; + MCFragment *Tail = nullptr; + }; + +private: + // At parse time, this holds the fragment list of the current subsection. At + // layout time, this holds the concatenated fragment lists of all subsections. + FragList *CurFragList; + // In many object file formats, this denotes the section symbol. In Mach-O, + // this denotes an optional temporary label at the section start. + MCSymbol *Begin; + MCSymbol *End = nullptr; + /// The alignment requirement of this section. + Align Alignment; + /// The section index in the assemblers section list. + unsigned Ordinal = 0; + + /// Whether this section has had instructions emitted into it. + bool HasInstructions : 1; + + bool IsRegistered : 1; + + bool IsText : 1; + bool IsBss : 1; + + /// Whether the section contains linker-relaxable fragments. If true, the + /// offset between two locations may not be fully resolved. + bool LinkerRelaxable : 1; + + MCFragment DummyFragment; + + // Mapping from subsection number to fragment list. At layout time, the + // subsection 0 list is replaced with concatenated fragments from all + // subsections. + SmallVector<std::pair<unsigned, FragList>, 1> Subsections; + + // Content and fixup storage for fragments + SmallVector<char, 0> ContentStorage; + SmallVector<MCFixup, 0> FixupStorage; + SmallVector<MCOperand, 0> MCOperandStorage; + +protected: + // TODO Make Name private when possible. + StringRef Name; + + MCSection(StringRef Name, bool IsText, bool IsBss, MCSymbol *Begin); + +public: + MCSection(const MCSection &) = delete; + MCSection &operator=(const MCSection &) = delete; + + StringRef getName() const { return Name; } + bool isText() const { return IsText; } + + MCSymbol *getBeginSymbol() { return Begin; } + const MCSymbol *getBeginSymbol() const { + return const_cast<MCSection *>(this)->getBeginSymbol(); + } + void setBeginSymbol(MCSymbol *Sym) { + assert(!Begin); + Begin = Sym; + } + MCSymbol *getEndSymbol(MCContext &Ctx); + bool hasEnded() const; + + Align getAlign() const { return Alignment; } + void setAlignment(Align Value) { Alignment = Value; } + + /// Makes sure that Alignment is at least MinAlignment. + void ensureMinAlignment(Align MinAlignment) { + if (Alignment < MinAlignment) + Alignment = MinAlignment; + } + + unsigned getOrdinal() const { return Ordinal; } + void setOrdinal(unsigned Value) { Ordinal = Value; } + + bool hasInstructions() const { return HasInstructions; } + void setHasInstructions(bool Value) { HasInstructions = Value; } + + bool isRegistered() const { return IsRegistered; } + void setIsRegistered(bool Value) { IsRegistered = Value; } + + bool isLinkerRelaxable() const { return LinkerRelaxable; } + void setLinkerRelaxable() { LinkerRelaxable = true; } + + MCFragment &getDummyFragment() { return DummyFragment; } + + FragList *curFragList() const { return CurFragList; } + iterator begin() const { return iterator(CurFragList->Head); } + iterator end() const { return {}; } + + void dump(DenseMap<const MCFragment *, SmallVector<const MCSymbol *, 0>> + *FragToSyms = nullptr) const; + + /// Check whether this section is "virtual", that is has no actual object + /// file contents. + bool isBssSection() const { return IsBss; } +}; + +inline MutableArrayRef<char> MCFragment::getContents() { + return {reinterpret_cast<char *>(this + 1), FixedSize}; +} +inline ArrayRef<char> MCFragment::getContents() const { + return {reinterpret_cast<const char *>(this + 1), FixedSize}; +} + +inline MutableArrayRef<char> MCFragment::getVarContents() { + return MutableArrayRef(getParent()->ContentStorage) + .slice(VarContentStart, VarContentEnd - VarContentStart); +} +inline ArrayRef<char> MCFragment::getVarContents() const { + return ArrayRef(getParent()->ContentStorage) + .slice(VarContentStart, VarContentEnd - VarContentStart); +} + +//== Fixup-related functions manage parent's storage using FixupStart and +// FixupSize. +inline MutableArrayRef<MCFixup> MCFragment::getFixups() { + return MutableArrayRef(getParent()->FixupStorage) + .slice(FixupStart, FixupEnd - FixupStart); +} +inline ArrayRef<MCFixup> MCFragment::getFixups() const { + return ArrayRef(getParent()->FixupStorage) + .slice(FixupStart, FixupEnd - FixupStart); +} + +inline MutableArrayRef<MCFixup> MCFragment::getVarFixups() { + return MutableArrayRef(getParent()->FixupStorage) + .slice(VarFixupStart, VarFixupEnd - VarFixupStart); +} +inline ArrayRef<MCFixup> MCFragment::getVarFixups() const { + return ArrayRef(getParent()->FixupStorage) + .slice(VarFixupStart, VarFixupEnd - VarFixupStart); +} + +//== FT_Relaxable functions +inline ArrayRef<MCOperand> MCFragment::getOperands() const { + assert(Kind == FT_Relaxable); + return MutableArrayRef(getParent()->MCOperandStorage) + .slice(u.relax.OperandStart, u.relax.OperandSize); +} +inline MCInst MCFragment::getInst() const { + assert(Kind == FT_Relaxable); + MCInst Inst; + Inst.setOpcode(u.relax.Opcode); + Inst.setFlags(u.relax.Flags); + Inst.setOperands(ArrayRef(getParent()->MCOperandStorage) + .slice(u.relax.OperandStart, u.relax.OperandSize)); + return Inst; +} +inline void MCFragment::setInst(const MCInst &Inst) { + assert(Kind == FT_Relaxable); + u.relax.Opcode = Inst.getOpcode(); + u.relax.Flags = Inst.getFlags(); + auto &S = getParent()->MCOperandStorage; + if (Inst.getNumOperands() > u.relax.OperandSize) { + u.relax.OperandStart = S.size(); + S.resize_for_overwrite(S.size() + Inst.getNumOperands()); + } + u.relax.OperandSize = Inst.getNumOperands(); + llvm::copy(Inst, S.begin() + u.relax.OperandStart); +} + inline MCSection::iterator &MCSection::iterator::operator++() { F = F->Next; return *this; diff --git a/llvm/include/llvm/MC/MCSectionCOFF.h b/llvm/include/llvm/MC/MCSectionCOFF.h index 4472a12..71efc41 100644 --- a/llvm/include/llvm/MC/MCSectionCOFF.h +++ b/llvm/include/llvm/MC/MCSectionCOFF.h @@ -51,11 +51,12 @@ class MCSectionCOFF final : public MCSection { private: friend class MCContext; + friend class MCAsmInfoCOFF; // The storage of Name is owned by MCContext's COFFUniquingMap. MCSectionCOFF(StringRef Name, unsigned Characteristics, MCSymbol *COMDATSymbol, int Selection, unsigned UniqueID, MCSymbol *Begin) - : MCSection(SV_COFF, Name, Characteristics & COFF::IMAGE_SCN_CNT_CODE, + : MCSection(Name, Characteristics & COFF::IMAGE_SCN_CNT_CODE, Characteristics & COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA, Begin), Characteristics(Characteristics), COMDATSymbol(COMDATSymbol), @@ -67,7 +68,7 @@ private: public: /// Decides whether a '.section' directive should be printed before the /// section name - bool shouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; + bool shouldOmitSectionDirective(StringRef Name) const; unsigned getCharacteristics() const { return Characteristics; } MCSymbol *getCOMDATSymbol() const { return COMDATSymbol; } @@ -78,12 +79,6 @@ public: bool isUnique() const { return UniqueID != NonUniqueID; } unsigned getUniqueID() const { return UniqueID; } - void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, - raw_ostream &OS, - uint32_t Subsection) const override; - bool useCodeAlign() const override; - StringRef getVirtualSectionKind() const override; - unsigned getOrAssignWinCFISectionID(unsigned *NextID) const { if (WinCFISectionID == ~0U) WinCFISectionID = (*NextID)++; @@ -93,8 +88,6 @@ public: static bool isImplicitlyDiscardable(StringRef Name) { return Name.starts_with(".debug"); } - - static bool classof(const MCSection *S) { return S->getVariant() == SV_COFF; } }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCSectionDXContainer.h b/llvm/include/llvm/MC/MCSectionDXContainer.h index 723b477..7d8e0c5 100644 --- a/llvm/include/llvm/MC/MCSectionDXContainer.h +++ b/llvm/include/llvm/MC/MCSectionDXContainer.h @@ -24,13 +24,7 @@ class MCSectionDXContainer final : public MCSection { friend class MCContext; MCSectionDXContainer(StringRef Name, SectionKind K, MCSymbol *Begin) - : MCSection(SV_DXContainer, Name, K.isText(), /*IsVirtual=*/false, - Begin) {} - -public: - void printSwitchToSection(const MCAsmInfo &, const Triple &, raw_ostream &, - uint32_t) const override; - bool useCodeAlign() const override { return false; } + : MCSection(Name, K.isText(), /*IsVirtual=*/false, Begin) {} }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCSectionELF.h b/llvm/include/llvm/MC/MCSectionELF.h index f09d305..f089dd9 100644 --- a/llvm/include/llvm/MC/MCSectionELF.h +++ b/llvm/include/llvm/MC/MCSectionELF.h @@ -52,14 +52,15 @@ class MCSectionELF final : public MCSection { private: friend class MCContext; + friend class MCAsmInfoELF; // The storage of Name is owned by MCContext's ELFUniquingMap. MCSectionELF(StringRef Name, unsigned type, unsigned flags, unsigned entrySize, const MCSymbolELF *group, bool IsComdat, unsigned UniqueID, MCSymbol *Begin, const MCSymbolELF *LinkedToSym) - : MCSection(SV_ELF, Name, flags & ELF::SHF_EXECINSTR, - type == ELF::SHT_NOBITS, Begin), + : MCSection(Name, flags & ELF::SHF_EXECINSTR, type == ELF::SHT_NOBITS, + Begin), Type(type), Flags(flags), UniqueID(UniqueID), EntrySize(entrySize), Group(group, IsComdat), LinkedToSym(LinkedToSym) { assert((!(Flags & ELF::SHF_GROUP) || Group.getPointer()) && @@ -68,15 +69,7 @@ private: Group.getPointer()->setIsSignature(); } - // TODO Delete after we stop supporting generation of GNU-style .zdebug_* - // sections. - void setSectionName(StringRef Name) { this->Name = Name; } - public: - /// Decides whether a '.section' directive should be printed before the - /// section name - bool shouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; - unsigned getType() const { return Type; } unsigned getFlags() const { return Flags; } unsigned getEntrySize() const { return EntrySize; } @@ -84,12 +77,6 @@ public: const MCSymbolELF *getGroup() const { return Group.getPointer(); } bool isComdat() const { return Group.getInt(); } - void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, - raw_ostream &OS, - uint32_t Subsection) const override; - bool useCodeAlign() const override; - StringRef getVirtualSectionKind() const override; - bool isUnique() const { return UniqueID != NonUniqueID; } unsigned getUniqueID() const { return UniqueID; } @@ -105,10 +92,6 @@ public: std::pair<uint64_t, uint64_t> getOffsets() const { return std::make_pair(StartOffset, EndOffset); } - - static bool classof(const MCSection *S) { - return S->getVariant() == SV_ELF; - } }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCSectionGOFF.h b/llvm/include/llvm/MC/MCSectionGOFF.h index 9e3f95e..2136148 100644 --- a/llvm/include/llvm/MC/MCSectionGOFF.h +++ b/llvm/include/llvm/MC/MCSectionGOFF.h @@ -52,36 +52,28 @@ class LLVM_ABI MCSectionGOFF final : public MCSection { mutable unsigned Emitted : 1; friend class MCContext; + friend class MCAsmInfoGOFF; friend class MCSymbolGOFF; MCSectionGOFF(StringRef Name, SectionKind K, bool IsVirtual, GOFF::SDAttr SDAttributes, MCSectionGOFF *Parent) - : MCSection(SV_GOFF, Name, K.isText(), IsVirtual, nullptr), - Parent(Parent), SDAttributes(SDAttributes), - SymbolType(GOFF::ESD_ST_SectionDefinition), IsBSS(K.isBSS()), - RequiresNonZeroLength(0), Emitted(0) {} + : MCSection(Name, K.isText(), IsVirtual, nullptr), Parent(Parent), + SDAttributes(SDAttributes), SymbolType(GOFF::ESD_ST_SectionDefinition), + IsBSS(K.isBSS()), RequiresNonZeroLength(0), Emitted(0) {} MCSectionGOFF(StringRef Name, SectionKind K, bool IsVirtual, GOFF::EDAttr EDAttributes, MCSectionGOFF *Parent) - : MCSection(SV_GOFF, Name, K.isText(), IsVirtual, nullptr), - Parent(Parent), EDAttributes(EDAttributes), - SymbolType(GOFF::ESD_ST_ElementDefinition), IsBSS(K.isBSS()), - RequiresNonZeroLength(0), Emitted(0) {} + : MCSection(Name, K.isText(), IsVirtual, nullptr), Parent(Parent), + EDAttributes(EDAttributes), SymbolType(GOFF::ESD_ST_ElementDefinition), + IsBSS(K.isBSS()), RequiresNonZeroLength(0), Emitted(0) {} MCSectionGOFF(StringRef Name, SectionKind K, bool IsVirtual, GOFF::PRAttr PRAttributes, MCSectionGOFF *Parent) - : MCSection(SV_GOFF, Name, K.isText(), IsVirtual, nullptr), - Parent(Parent), PRAttributes(PRAttributes), - SymbolType(GOFF::ESD_ST_PartReference), IsBSS(K.isBSS()), - RequiresNonZeroLength(0), Emitted(0) {} + : MCSection(Name, K.isText(), IsVirtual, nullptr), Parent(Parent), + PRAttributes(PRAttributes), SymbolType(GOFF::ESD_ST_PartReference), + IsBSS(K.isBSS()), RequiresNonZeroLength(0), Emitted(0) {} public: - void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, - raw_ostream &OS, - uint32_t Subsection) const override; - - bool useCodeAlign() const override { return false; } - // Return the parent section. MCSectionGOFF *getParent() const { return Parent; } @@ -111,7 +103,7 @@ public: // Returns the text style for a section. Only defined for ED and PR sections. GOFF::ESDTextStyle getTextStyle() const { - assert((isED() || isPR() || isVirtualSection()) && "Expect ED or PR section"); + assert((isED() || isPR() || isBssSection()) && "Expect ED or PR section"); if (isED()) return EDAttributes.TextStyle; if (isPR()) @@ -123,8 +115,6 @@ public: bool requiresNonZeroLength() const { return RequiresNonZeroLength; } void setName(StringRef SectionName) { Name = SectionName; } - - static bool classof(const MCSection *S) { return S->getVariant() == SV_GOFF; } }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCSectionMachO.h b/llvm/include/llvm/MC/MCSectionMachO.h index 4312175..a65d7e0 100644 --- a/llvm/include/llvm/MC/MCSectionMachO.h +++ b/llvm/include/llvm/MC/MCSectionMachO.h @@ -23,6 +23,8 @@ namespace llvm { /// This represents a section on a Mach-O system (used by Mac OS X). On a Mac /// system, these are also described in /usr/include/mach-o/loader.h. class LLVM_ABI MCSectionMachO final : public MCSection { + friend class MCContext; + friend class MCAsmInfoDarwin; char SegmentName[16]; // Not necessarily null terminated! /// This is the SECTION_TYPE and SECTION_ATTRIBUTES field of a section, drawn @@ -42,7 +44,6 @@ class LLVM_ABI MCSectionMachO final : public MCSection { MCSectionMachO(StringRef Segment, StringRef Section, unsigned TAA, unsigned reserved2, SectionKind K, MCSymbol *Begin); - friend class MCContext; public: StringRef getSegmentName() const { @@ -76,21 +77,12 @@ public: bool &TAAParsed, // Out. unsigned &StubSize); // Out. - void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, - raw_ostream &OS, - uint32_t Subsection) const override; - bool useCodeAlign() const override; - void allocAtoms(); const MCSymbol *getAtom(size_t I) const; void setAtom(size_t I, const MCSymbol *Sym); unsigned getLayoutOrder() const { return LayoutOrder; } void setLayoutOrder(unsigned Value) { LayoutOrder = Value; } - - static bool classof(const MCSection *S) { - return S->getVariant() == SV_MachO; - } }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCSectionSPIRV.h b/llvm/include/llvm/MC/MCSectionSPIRV.h index 091114a..6850965 100644 --- a/llvm/include/llvm/MC/MCSectionSPIRV.h +++ b/llvm/include/llvm/MC/MCSectionSPIRV.h @@ -18,22 +18,13 @@ namespace llvm { -class MCSymbol; - class MCSectionSPIRV final : public MCSection { friend class MCContext; MCSectionSPIRV() - : MCSection(SV_SPIRV, "", /*IsText=*/true, /*IsVirtual=*/false, + : MCSection("", /*IsText=*/true, /*IsVirtual=*/false, /*Begin=*/nullptr) {} // TODO: Add StringRef Name to MCSectionSPIRV. - -public: - ~MCSectionSPIRV() = default; - void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, - raw_ostream &OS, - uint32_t Subsection) const override {} - bool useCodeAlign() const override { return false; } }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCSectionWasm.h b/llvm/include/llvm/MC/MCSectionWasm.h index 4523818..5ec01ed 100644 --- a/llvm/include/llvm/MC/MCSectionWasm.h +++ b/llvm/include/llvm/MC/MCSectionWasm.h @@ -49,26 +49,18 @@ class MCSectionWasm final : public MCSection { // The storage of Name is owned by MCContext's WasmUniquingMap. friend class MCContext; + friend class MCAsmInfoWasm; MCSectionWasm(StringRef Name, SectionKind K, unsigned SegmentFlags, const MCSymbolWasm *Group, unsigned UniqueID, MCSymbol *Begin) - : MCSection(SV_Wasm, Name, K.isText(), /*IsVirtual=*/false, Begin), + : MCSection(Name, K.isText(), /*IsVirtual=*/false, Begin), UniqueID(UniqueID), Group(Group), IsWasmData(K.isReadOnly() || K.isWriteable()), IsMetadata(K.isMetadata()), SegmentFlags(SegmentFlags) {} public: - /// Decides whether a '.section' directive should be printed before the - /// section name - bool shouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; - const MCSymbolWasm *getGroup() const { return Group; } unsigned getSegmentFlags() const { return SegmentFlags; } - void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, - raw_ostream &OS, - uint32_t Subsection) const override; - bool useCodeAlign() const override; - bool isWasmData() const { return IsWasmData; } bool isMetadata() const { return IsMetadata; } @@ -89,7 +81,6 @@ public: assert(isWasmData()); IsPassive = V; } - static bool classof(const MCSection *S) { return S->getVariant() == SV_Wasm; } }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCSectionXCOFF.h b/llvm/include/llvm/MC/MCSectionXCOFF.h index 499df6b5..0571f95 100644 --- a/llvm/include/llvm/MC/MCSectionXCOFF.h +++ b/llvm/include/llvm/MC/MCSectionXCOFF.h @@ -31,6 +31,7 @@ namespace llvm { // implemented yet. class MCSectionXCOFF final : public MCSection { friend class MCContext; + friend class MCAsmInfoXCOFF; std::optional<XCOFF::CsectProperties> CsectProp; MCSymbolXCOFF *const QualName; @@ -46,7 +47,7 @@ class MCSectionXCOFF final : public MCSection { XCOFF::SymbolType ST, SectionKind K, MCSymbolXCOFF *QualName, MCSymbol *Begin, StringRef SymbolTableName, bool MultiSymbolsAllowed) - : MCSection(SV_XCOFF, Name, K.isText(), + : MCSection(Name, K.isText(), /*IsVirtual=*/ST == XCOFF::XTY_CM && SMC != XCOFF::XMC_TD, Begin), CsectProp(XCOFF::CsectProperties(SMC, ST)), QualName(QualName), @@ -77,7 +78,7 @@ class MCSectionXCOFF final : public MCSection { XCOFF::DwarfSectionSubtypeFlags DwarfSubtypeFlags, MCSymbol *Begin, StringRef SymbolTableName, bool MultiSymbolsAllowed) - : MCSection(SV_XCOFF, Name, K.isText(), /*IsVirtual=*/false, Begin), + : MCSection(Name, K.isText(), /*IsVirtual=*/false, Begin), QualName(QualName), SymbolTableName(SymbolTableName), DwarfSubtypeFlags(DwarfSubtypeFlags), MultiSymbolsAllowed(MultiSymbolsAllowed), Kind(K) { @@ -95,10 +96,6 @@ class MCSectionXCOFF final : public MCSection { public: ~MCSectionXCOFF(); - static bool classof(const MCSection *S) { - return S->getVariant() == SV_XCOFF; - } - XCOFF::StorageMappingClass getMappingClass() const { assert(isCsect() && "Only csect section has mapping class property!"); return CsectProp->MappingClass; @@ -115,10 +112,6 @@ public: } MCSymbolXCOFF *getQualNameSymbol() const { return QualName; } - void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, - raw_ostream &OS, - uint32_t Subsection) const override; - bool useCodeAlign() const override; StringRef getSymbolTableName() const { return SymbolTableName; } void setSymbolTableName(StringRef STN) { SymbolTableName = STN; } bool isMultiSymbolsAllowed() const { return MultiSymbolsAllowed; } diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index 4b91dbc..79c715e 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -259,6 +259,8 @@ class LLVM_ABI MCStreamer { bool AllowAutoPadding = false; protected: + bool IsObj = false; + // Symbol of the current epilog for which we are processing SEH directives. WinEH::FrameInfo::Epilog *CurrentWinEpilog = nullptr; @@ -270,6 +272,8 @@ protected: /// section changes. virtual void changeSection(MCSection *, uint32_t); + void addFragment(MCFragment *F); + virtual void emitCFIStartProcImpl(MCDwarfFrameInfo &Frame); virtual void emitCFIEndProcImpl(MCDwarfFrameInfo &CurFrame); @@ -308,6 +312,7 @@ public: virtual void reset(); MCContext &getContext() const { return Context; } + bool isObj() const { return IsObj; } // MCObjectStreamer has an MCAssembler and allows more expression folding at // parse time. @@ -425,10 +430,15 @@ public: } MCFragment *getCurrentFragment() const { + // Ensure consistency with the section stack. assert(!getCurrentSection().first || CurFrag->getParent() == getCurrentSection().first); + // Ensure we eagerly allocate an empty fragment after adding fragment with a + // variable-size tail. + assert(!CurFrag || CurFrag->getKind() == MCFragment::FT_Data); return CurFrag; } + size_t getCurFragSize() const { return getCurrentFragment()->getFixedSize(); } /// Save the current and previous section on the section stack. void pushSection() { SectionStack.push_back( @@ -449,16 +459,13 @@ public: bool switchSection(MCSection *Section, const MCExpr *); /// Similar to switchSection, but does not print the section directive. - virtual void switchSectionNoPrint(MCSection *Section); + void switchSectionNoPrint(MCSection *Section); /// Create the default sections and set the initial one. virtual void initSections(bool NoExecStack, const MCSubtargetInfo &STI); MCSymbol *endSection(MCSection *Section); - void insert(MCFragment *F); - void newFragment(); - /// Returns the mnemonic for \p MI, if the streamer has access to a /// instruction printer and returns an empty string otherwise. virtual StringRef getMnemonic(const MCInst &MI) const { return ""; } @@ -979,7 +986,7 @@ public: const MCSymbol *Lo); virtual MCSymbol *getDwarfLineTableSymbol(unsigned CUID); - virtual void emitCFISections(bool EH, bool Debug); + virtual void emitCFISections(bool EH, bool Debug, bool SFrame); void emitCFIStartProc(bool IsSimple, SMLoc Loc = SMLoc()); void emitCFIEndProc(); virtual void emitCFIDefCfa(int64_t Register, int64_t Offset, SMLoc Loc = {}); diff --git a/llvm/include/llvm/MC/MCSymbolELF.h b/llvm/include/llvm/MC/MCSymbolELF.h index eba9964..7c271e7 100644 --- a/llvm/include/llvm/MC/MCSymbolELF.h +++ b/llvm/include/llvm/MC/MCSymbolELF.h @@ -13,6 +13,7 @@ namespace llvm { class MCSymbolELF : public MCSymbol { + friend class MCAsmInfoELF; /// An expression describing how to calculate the size of a symbol. If a /// symbol has no size this field will be NULL. const MCExpr *SymbolSize = nullptr; diff --git a/llvm/include/llvm/MC/MCTargetOptions.h b/llvm/include/llvm/MC/MCTargetOptions.h index d95adf9..235d58d 100644 --- a/llvm/include/llvm/MC/MCTargetOptions.h +++ b/llvm/include/llvm/MC/MCTargetOptions.h @@ -102,6 +102,9 @@ public: // functions on Darwins. bool EmitCompactUnwindNonCanonical : 1; + // Whether to emit SFrame unwind sections. + bool EmitSFrameUnwind : 1; + // Whether or not to use full register names on PowerPC. bool PPCUseFullRegisterNames : 1; diff --git a/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h b/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h index b057eff..adfdccd 100644 --- a/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h +++ b/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h @@ -40,6 +40,8 @@ LLVM_ABI EmitDwarfUnwindType getEmitDwarfUnwind(); LLVM_ABI bool getEmitCompactUnwindNonCanonical(); +LLVM_ABI bool getEmitSFrameUnwind(); + LLVM_ABI bool getShowMCInst(); LLVM_ABI bool getFatalWarnings(); diff --git a/llvm/include/llvm/MC/MCXCOFFStreamer.h b/llvm/include/llvm/MC/MCXCOFFStreamer.h index 870d48f..c3bc2ca9 100644 --- a/llvm/include/llvm/MC/MCXCOFFStreamer.h +++ b/llvm/include/llvm/MC/MCXCOFFStreamer.h @@ -22,6 +22,7 @@ public: XCOFFObjectWriter &getWriter(); + void changeSection(MCSection *Section, uint32_t Subsection = 0) override; bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override; void emitCommonSymbol(MCSymbol *Symbol, uint64_t Size, Align ByteAlignment) override; diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h index a3aa0d9..ced1afd 100644 --- a/llvm/include/llvm/Object/ELFObjectFile.h +++ b/llvm/include/llvm/Object/ELFObjectFile.h @@ -1479,6 +1479,7 @@ template <class ELFT> Triple::OSType ELFObjectFile<ELFT>::getOS() const { case ELF::ELFOSABI_OPENBSD: return Triple::OpenBSD; case ELF::ELFOSABI_CUDA: + case ELF::ELFOSABI_CUDA_V2: return Triple::CUDA; case ELF::ELFOSABI_AMDGPU_HSA: return Triple::AMDHSA; diff --git a/llvm/include/llvm/Object/SFrameParser.h b/llvm/include/llvm/Object/SFrameParser.h new file mode 100644 index 0000000..245e7ba --- /dev/null +++ b/llvm/include/llvm/Object/SFrameParser.h @@ -0,0 +1,65 @@ +//===- SFrameParser.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECT_SFRAME_H +#define LLVM_OBJECT_SFRAME_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/BinaryFormat/SFrame.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Error.h" +#include <cstdint> + +namespace llvm { +namespace object { + +template <endianness E> class SFrameParser { +public: + static Expected<SFrameParser> create(ArrayRef<uint8_t> Contents, + uint64_t SectionAddress); + + const sframe::Preamble<E> &getPreamble() const { return Header.Preamble; } + const sframe::Header<E> &getHeader() const { return Header; } + + Expected<ArrayRef<uint8_t>> getAuxHeader() const; + + bool usesFixedRAOffset() const { + return getHeader().ABIArch == sframe::ABI::AMD64EndianLittle; + } + bool usesFixedFPOffset() const { + return false; // Not used in any currently defined ABI. + } + + using FDERange = ArrayRef<sframe::FuncDescEntry<E>>; + Expected<FDERange> fdes() const; + + // Decodes the start address of the given FDE, which must be one of the + // objects returned by the `fdes()` function. + uint64_t getAbsoluteStartAddress(typename FDERange::iterator FDE) const; + +private: + ArrayRef<uint8_t> Data; + uint64_t SectionAddress; + const sframe::Header<E> &Header; + + SFrameParser(ArrayRef<uint8_t> Data, uint64_t SectionAddress, + const sframe::Header<E> &Header) + : Data(Data), SectionAddress(SectionAddress), Header(Header) {} + + uint64_t getFDEBase() const { + return sizeof(Header) + Header.AuxHdrLen + Header.FDEOff; + } +}; + +extern template class LLVM_TEMPLATE_ABI SFrameParser<endianness::big>; +extern template class LLVM_TEMPLATE_ABI SFrameParser<endianness::little>; + +} // end namespace object +} // end namespace llvm + +#endif // LLVM_OBJECT_SFRAME_H diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index b0360f1..97c3ff8 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -114,13 +114,16 @@ #include "llvm/Target/CGPassBuilderOption.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/CFGuard.h" +#include "llvm/Transforms/ObjCARC.h" #include "llvm/Transforms/Scalar/ConstantHoisting.h" #include "llvm/Transforms/Scalar/LoopPassManager.h" #include "llvm/Transforms/Scalar/LoopStrengthReduce.h" +#include "llvm/Transforms/Scalar/LoopTermFold.h" #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h" #include "llvm/Transforms/Scalar/MergeICmps.h" #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h" +#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h" #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" #include "llvm/Transforms/Utils/LowerInvoke.h" #include <cassert> @@ -754,7 +757,12 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addIRPasses( // Run loop strength reduction before anything else. if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) { - addPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(), + LoopPassManager LPM; + LPM.addPass(CanonicalizeFreezeInLoopsPass()); + LPM.addPass(LoopStrengthReducePass()); + if (Opt.EnableLoopTermFold) + LPM.addPass(LoopTermFoldPass()); + addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/true)); } @@ -799,7 +807,8 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addIRPasses( addPass(ScalarizeMaskedMemIntrinPass()); // Expand reduction intrinsics into shuffle sequences if the target wants to. - addPass(ExpandReductionsPass()); + if (!Opt.DisableExpandReductions) + addPass(ExpandReductionsPass()); // Convert conditional moves to conditional jumps when profitable. if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize) @@ -877,6 +886,9 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addISelPrepare( if (Opt.RequiresCodeGenSCCOrder) addPass.requireCGSCCOrder(); + if (getOptLevel() != CodeGenOptLevel::None) + addPass(ObjCARCContractPass()); + addPass(CallBrPreparePass()); // Add both the safe stack and the stack protection passes: each of them will // only protect functions that have corresponding attributes. diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index 732fdc7..bee2106 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -113,6 +113,7 @@ MACHINE_FUNCTION_PASS("early-machinelicm", EarlyMachineLICMPass()) MACHINE_FUNCTION_PASS("early-tailduplication", EarlyTailDuplicatePass()) MACHINE_FUNCTION_PASS("fentry-insert", FEntryInserterPass()) MACHINE_FUNCTION_PASS("finalize-isel", FinalizeISelPass()) +MACHINE_FUNCTION_PASS("finalizebundle-test", FinalizeBundleTestPass()) MACHINE_FUNCTION_PASS("fixup-statepoint-caller-saved", FixupStatepointCallerSavedPass()) MACHINE_FUNCTION_PASS("init-undef", InitUndefPass()) MACHINE_FUNCTION_PASS("localstackalloc", LocalStackSlotAllocationPass()) diff --git a/llvm/include/llvm/Support/AArch64AttributeParser.h b/llvm/include/llvm/Support/AArch64AttributeParser.h index aa82ca1..f4552ef 100644 --- a/llvm/include/llvm/Support/AArch64AttributeParser.h +++ b/llvm/include/llvm/Support/AArch64AttributeParser.h @@ -25,6 +25,17 @@ public: : ELFExtendedAttrParser(nullptr, returnTagsNamesMap()) {} }; +// Used for extracting AArch64 Build Attributes +struct AArch64BuildAttrSubsections { + struct PauthSubSection { + uint64_t TagPlatform = 0; + uint64_t TagSchema = 0; + } Pauth; + uint32_t AndFeatures = 0; +}; + +LLVM_ABI AArch64BuildAttrSubsections +extractBuildAttributesSubsections(const llvm::AArch64AttributeParser &); } // namespace llvm #endif // LLVM_SUPPORT_AARCH64ATTRIBUTEPARSER_H diff --git a/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h b/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h index a119b07..8f36739 100644 --- a/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h +++ b/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h @@ -223,7 +223,8 @@ enum : int32_t { KERNEL_CODE_PROPERTY(ENABLE_SGPR_DISPATCH_ID, 4, 1), KERNEL_CODE_PROPERTY(ENABLE_SGPR_FLAT_SCRATCH_INIT, 5, 1), KERNEL_CODE_PROPERTY(ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, 6, 1), - KERNEL_CODE_PROPERTY(RESERVED0, 7, 3), + KERNEL_CODE_PROPERTY(RESERVED0, 7, 2), + KERNEL_CODE_PROPERTY(USES_CU_STORES, 9, 1), // GFX12.5 +cu-stores KERNEL_CODE_PROPERTY(ENABLE_WAVEFRONT_SIZE32, 10, 1), // GFX10+ KERNEL_CODE_PROPERTY(USES_DYNAMIC_STACK, 11, 1), KERNEL_CODE_PROPERTY(RESERVED1, 12, 4), diff --git a/llvm/include/llvm/Support/AlwaysTrue.h b/llvm/include/llvm/Support/AlwaysTrue.h new file mode 100644 index 0000000..b696856 --- /dev/null +++ b/llvm/include/llvm/Support/AlwaysTrue.h @@ -0,0 +1,25 @@ +//===--- AlwaysTrue.h - Helper for oqaque truthy values --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_ALWAYS_TRUE_H +#define LLVM_SUPPORT_ALWAYS_TRUE_H + +#include <cstdlib> + +namespace llvm { +inline bool getNonFoldableAlwaysTrue() { + // Some parts of the codebase require a "constant true value" used as a + // predicate. These cases require that even with LTO and static linking, + // it's not possible for the compiler to fold the value. As compilers + // aren't smart enough to know that getenv() never returns -1, this will do + // the job. + return std::getenv("LLVM_IGNORED_ENV_VAR") != (char *)-1; +} +} // end namespace llvm + +#endif // LLVM_SUPPORT_ALWAYS_TRUE_H diff --git a/llvm/include/llvm/Support/CommandLine.h b/llvm/include/llvm/Support/CommandLine.h index adaa75c..ca725b8 100644 --- a/llvm/include/llvm/Support/CommandLine.h +++ b/llvm/include/llvm/Support/CommandLine.h @@ -1518,11 +1518,18 @@ public: [](const typename ParserClass::parser_data_type &) {}; }; -extern template class opt<unsigned>; -extern template class opt<int>; -extern template class opt<std::string>; -extern template class opt<char>; -extern template class opt<bool>; +#if !(defined(LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS) && defined(_MSC_VER)) +// Only instantiate opt<std::string> when not building a Windows DLL. When +// exporting opt<std::string>, MSVC implicitly exports symbols for +// std::basic_string through transitive inheritance via std::string. These +// symbols may appear in clients, leading to duplicate symbol conflicts. +extern template class LLVM_TEMPLATE_ABI opt<std::string>; +#endif + +extern template class LLVM_TEMPLATE_ABI opt<unsigned>; +extern template class LLVM_TEMPLATE_ABI opt<int>; +extern template class LLVM_TEMPLATE_ABI opt<char>; +extern template class LLVM_TEMPLATE_ABI opt<bool>; //===----------------------------------------------------------------------===// // Default storage class definition: external storage. This implementation diff --git a/llvm/include/llvm/Support/Debug.h b/llvm/include/llvm/Support/Debug.h index 924d7b2..5542089 100644 --- a/llvm/include/llvm/Support/Debug.h +++ b/llvm/include/llvm/Support/Debug.h @@ -39,13 +39,19 @@ class raw_ostream; /// isCurrentDebugType - Return true if the specified string is the debug type /// specified on the command line, or if none was specified on the command line /// with the -debug-only=X option. -/// -bool isCurrentDebugType(const char *Type); +/// An optional level can be provided to control the verbosity of the output. +/// If the provided level is not 0 and user specified a level below the provided +/// level, return false. +bool isCurrentDebugType(const char *Type, int Level = 0); /// setCurrentDebugType - Set the current debug type, as if the -debug-only=X /// option were specified. Note that DebugFlag also needs to be set to true for /// debug output to be produced. -/// +/// The debug type format is "type[:level]", where the level is an optional +/// integer. If a level is provided, the debug output is enabled only if the +/// user specified a level at least as high as the provided level. +/// 0 is a special level that acts as an opt-out for this specific debug type +/// without affecting the other debug output. void setCurrentDebugType(const char *Type); /// setCurrentDebugTypes - Set the current debug type, as if the diff --git a/llvm/include/llvm/Support/DebugLog.h b/llvm/include/llvm/Support/DebugLog.h new file mode 100644 index 0000000..8fca2d5 --- /dev/null +++ b/llvm/include/llvm/Support/DebugLog.h @@ -0,0 +1,175 @@ +//===- llvm/Support/DebugLog.h - Logging like debug output ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// This file contains macros for logging like debug output. It builds upon the +// support in Debug.h but provides a utility function for common debug output +// style. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_DEBUGLOG_H +#define LLVM_SUPPORT_DEBUGLOG_H + +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +#ifndef NDEBUG + +// LDBG() is a macro that can be used as a raw_ostream for debugging. +// It will stream the output to the dbgs() stream, with a prefix of the +// debug type and the file and line number. A trailing newline is added to the +// output automatically. If the streamed content contains a newline, the prefix +// is added to each beginning of a new line. Nothing is printed if the debug +// output is not enabled or the debug type does not match. +// +// E.g., +// LDBG() << "Bitset contains: " << Bitset; +// is somehow equivalent to +// LLVM_DEBUG(dbgs() << "[" << DEBUG_TYPE << "] " << __FILE__ << ":" << +// __LINE__ << " " +// << "Bitset contains: " << Bitset << "\n"); +// +// An optional `level` argument can be provided to control the verbosity of the +// output. The default level is 1, and is in increasing level of verbosity. +// +// The `level` argument can be a literal integer, or a macro that evaluates to +// an integer. +// +#define LDBG(...) _GET_LDBG_MACRO(__VA_ARGS__)(__VA_ARGS__) + +// Helper macros to choose the correct macro based on the number of arguments. +#define LDBG_FUNC_CHOOSER(_f1, _f2, ...) _f2 +#define LDBG_FUNC_RECOMPOSER(argsWithParentheses) \ + LDBG_FUNC_CHOOSER argsWithParentheses +#define LDBG_CHOOSE_FROM_ARG_COUNT(...) \ + LDBG_FUNC_RECOMPOSER((__VA_ARGS__, LDBG_LOG_LEVEL, )) +#define LDBG_NO_ARG_EXPANDER() , LDBG_LOG_LEVEL_1 +#define _GET_LDBG_MACRO(...) \ + LDBG_CHOOSE_FROM_ARG_COUNT(LDBG_NO_ARG_EXPANDER __VA_ARGS__()) + +// Dispatch macros to support the `level` argument or none (default to 1) +#define LDBG_LOG_LEVEL(LEVEL) \ + DEBUGLOG_WITH_STREAM_AND_TYPE(llvm::dbgs(), LEVEL, DEBUG_TYPE) +#define LDBG_LOG_LEVEL_1() LDBG_LOG_LEVEL(1) + +#define DEBUGLOG_WITH_STREAM_TYPE_FILE_AND_LINE(STREAM, LEVEL, TYPE, FILE, \ + LINE) \ + for (bool _c = \ + (::llvm::DebugFlag && ::llvm::isCurrentDebugType(TYPE, LEVEL)); \ + _c; _c = false) \ + ::llvm::impl::raw_ldbg_ostream{ \ + ::llvm::impl::computePrefix(TYPE, FILE, LINE, LEVEL), (STREAM)} \ + .asLvalue() + +#define DEBUGLOG_WITH_STREAM_TYPE_AND_FILE(STREAM, LEVEL, TYPE, FILE) \ + DEBUGLOG_WITH_STREAM_TYPE_FILE_AND_LINE(STREAM, LEVEL, TYPE, FILE, __LINE__) +// When __SHORT_FILE__ is not defined, the File is the full path, +// otherwise __SHORT_FILE__ is defined in CMake to provide the file name +// without the path prefix. +#if defined(__SHORT_FILE__) +#define DEBUGLOG_WITH_STREAM_AND_TYPE(STREAM, LEVEL, TYPE) \ + DEBUGLOG_WITH_STREAM_TYPE_AND_FILE(STREAM, LEVEL, TYPE, __SHORT_FILE__) +#else +#define DEBUGLOG_WITH_STREAM_AND_TYPE(STREAM, LEVEL, TYPE) \ + DEBUGLOG_WITH_STREAM_TYPE_AND_FILE(STREAM, LEVEL, TYPE, \ + ::llvm::impl::getShortFileName(__FILE__)) +#endif + +namespace impl { + +/// A raw_ostream that tracks `\n` and print the prefix. +class LLVM_ABI raw_ldbg_ostream final : public raw_ostream { + std::string Prefix; + raw_ostream &Os; + bool HasPendingNewline = true; + + /// Split the line on newlines and insert the prefix before each newline. + /// Forward everything to the underlying stream. + void write_impl(const char *Ptr, size_t Size) final { + auto Str = StringRef(Ptr, Size); + // Handle the initial prefix. + if (!Str.empty()) + writeWithPrefix(StringRef()); + + auto Eol = Str.find('\n'); + while (Eol != StringRef::npos) { + StringRef Line = Str.take_front(Eol + 1); + if (!Line.empty()) + writeWithPrefix(Line); + HasPendingNewline = true; + Str = Str.drop_front(Eol + 1); + Eol = Str.find('\n'); + } + if (!Str.empty()) + writeWithPrefix(Str); + } + void emitPrefix() { Os.write(Prefix.c_str(), Prefix.size()); } + void writeWithPrefix(StringRef Str) { + if (HasPendingNewline) { + emitPrefix(); + HasPendingNewline = false; + } + Os.write(Str.data(), Str.size()); + } + +public: + explicit raw_ldbg_ostream(std::string Prefix, raw_ostream &Os) + : Prefix(std::move(Prefix)), Os(Os) { + SetUnbuffered(); + } + ~raw_ldbg_ostream() final { + flushEol(); + Os << '\n'; + } + void flushEol() { + if (HasPendingNewline) { + emitPrefix(); + HasPendingNewline = false; + } + } + + /// Forward the current_pos method to the underlying stream. + uint64_t current_pos() const final { return Os.tell(); } + + /// Some of the `<<` operators expect an lvalue, so we trick the type system. + raw_ldbg_ostream &asLvalue() { return *this; } +}; + +/// Remove the path prefix from the file name. +static LLVM_ATTRIBUTE_UNUSED constexpr const char * +getShortFileName(const char *path) { + const char *filename = path; + for (const char *p = path; *p != '\0'; ++p) { + if (*p == '/' || *p == '\\') + filename = p + 1; + } + return filename; +} + +/// Compute the prefix for the debug log in the form of: +/// "[DebugType] File:Line " +/// Where the File is the file name without the path prefix. +static LLVM_ATTRIBUTE_UNUSED std::string +computePrefix(const char *DebugType, const char *File, int Line, int Level) { + std::string Prefix; + raw_string_ostream OsPrefix(Prefix); + if (DebugType) + OsPrefix << "[" << DebugType << ":" << Level << "] "; + OsPrefix << File << ":" << Line << " "; + return OsPrefix.str(); +} +} // end namespace impl +#else +// As others in Debug, When compiling without assertions, the -debug-* options +// and all inputs too LDBG() are ignored. +#define LDBG(...) \ + for (bool _c = false; _c; _c = false) \ + ::llvm::nulls() +#endif +} // end namespace llvm + +#endif // LLVM_SUPPORT_DEBUGLOG_H diff --git a/llvm/include/llvm/Support/ThreadPool.h b/llvm/include/llvm/Support/ThreadPool.h index 9272760..c26681c 100644 --- a/llvm/include/llvm/Support/ThreadPool.h +++ b/llvm/include/llvm/Support/ThreadPool.h @@ -149,10 +149,6 @@ public: /// number of threads! unsigned getMaxConcurrency() const override { return MaxThreadCount; } - // TODO: Remove, misleading legacy name warning! - LLVM_DEPRECATED("Use getMaxConcurrency instead", "getMaxConcurrency") - unsigned getThreadCount() const { return MaxThreadCount; } - /// Returns true if the current thread is a worker thread of this thread pool. bool isWorkerThread() const; @@ -233,10 +229,6 @@ public: /// Returns always 1: there is no concurrency. unsigned getMaxConcurrency() const override { return 1; } - // TODO: Remove, misleading legacy name warning! - LLVM_DEPRECATED("Use getMaxConcurrency instead", "getMaxConcurrency") - unsigned getThreadCount() const { return 1; } - /// Returns true if the current thread is a worker thread of this thread pool. bool isWorkerThread() const; diff --git a/llvm/include/llvm/Support/Windows/WindowsSupport.h b/llvm/include/llvm/Support/Windows/WindowsSupport.h index ffc6fdf..f35e7b5 100644 --- a/llvm/include/llvm/Support/Windows/WindowsSupport.h +++ b/llvm/include/llvm/Support/Windows/WindowsSupport.h @@ -245,6 +245,10 @@ LLVM_ABI std::error_code widenPath(const Twine &Path8, SmallVectorImpl<wchar_t> &Path16, size_t MaxPathLen = MAX_PATH); +/// Retrieves the handle to a in-memory system module such as ntdll.dll, while +/// ensuring we're not retrieving a malicious injected module but a module +/// loaded from the system path. +LLVM_ABI HMODULE loadSystemModuleSecure(LPCWSTR lpModuleName); } // end namespace windows } // end namespace sys } // end namespace llvm. diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h index a2b86eb..9d67d8b 100644 --- a/llvm/include/llvm/TableGen/Record.h +++ b/llvm/include/llvm/TableGen/Record.h @@ -841,6 +841,7 @@ public: SIZE, EMPTY, GETDAGOP, + GETDAGOPNAME, LOG2, REPR, LISTFLATTEN, @@ -910,6 +911,7 @@ public: GETDAGARG, GETDAGNAME, SETDAGOP, + SETDAGOPNAME }; private: diff --git a/llvm/include/llvm/Target/CGPassBuilderOption.h b/llvm/include/llvm/Target/CGPassBuilderOption.h index f29cbe7..8d0a7e6 100644 --- a/llvm/include/llvm/Target/CGPassBuilderOption.h +++ b/llvm/include/llvm/Target/CGPassBuilderOption.h @@ -52,6 +52,8 @@ struct CGPassBuilderOption { bool EnableMachineFunctionSplitter = false; bool EnableSinkAndFold = false; bool EnableTailMerge = true; + /// Enable LoopTermFold immediately after LSR. + bool EnableLoopTermFold = false; bool MISchedPostRA = false; bool EarlyLiveIntervals = false; bool GCEmptyBlocks = false; diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index fc81ab7..b619de3 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -194,6 +194,7 @@ def IsExact : MIFlagEnum<"IsExact">; def NoSWrap : MIFlagEnum<"NoSWrap">; def NoUWrap : MIFlagEnum<"NoUWrap">; def NonNeg : MIFlagEnum<"NonNeg">; +def InBounds : MIFlagEnum<"InBounds">; def MIFlags; // def not; -> Already defined as a SDNode diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index 7577792..b65a63b 100644 --- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -78,6 +78,8 @@ def : GINodeEquiv<G_XOR, xor>; def : GINodeEquiv<G_SHL, shl>; def : GINodeEquiv<G_LSHR, srl>; def : GINodeEquiv<G_ASHR, sra>; +def : GINodeEquiv<G_ABDS, abds>; +def : GINodeEquiv<G_ABDU, abdu>; def : GINodeEquiv<G_SADDSAT, saddsat>; def : GINodeEquiv<G_UADDSAT, uaddsat>; def : GINodeEquiv<G_SSUBSAT, ssubsat>; diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index f420798..db90f2e 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -133,10 +133,11 @@ public: EmitStackSizeSection(false), EnableMachineOutliner(false), EnableMachineFunctionSplitter(false), EnableStaticDataPartitioning(false), SupportsDefaultOutlining(false), - EmitAddrsig(false), BBAddrMap(false), EmitCallSiteInfo(false), - SupportsDebugEntryValues(false), EnableDebugEntryValues(false), - ValueTrackingVariableLocations(false), ForceDwarfFrameSection(false), - XRayFunctionIndex(true), DebugStrictDwarf(false), Hotpatch(false), + EmitAddrsig(false), BBAddrMap(false), EmitCallGraphSection(false), + EmitCallSiteInfo(false), SupportsDebugEntryValues(false), + EnableDebugEntryValues(false), ValueTrackingVariableLocations(false), + ForceDwarfFrameSection(false), XRayFunctionIndex(true), + DebugStrictDwarf(false), Hotpatch(false), PPCGenScalarMASSEntries(false), JMCInstrument(false), EnableCFIFixup(false), MisExpect(false), XCOFFReadOnlyPointers(false), VerifyArgABICompliance(true), @@ -319,6 +320,9 @@ public: /// to selectively generate basic block sections. std::shared_ptr<MemoryBuffer> BBSectionsFuncListBuf; + /// Emit section containing call graph metadata. + unsigned EmitCallGraphSection : 1; + /// The flag enables call site info production. It is used only for debug /// info, and it is restricted only to optimized code. This can be used for /// something else, so that should be controlled in the frontend. diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index 8ec8697..a4ed62b 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -1147,6 +1147,9 @@ def fadd_contract : PatFrag<(ops node:$a, node:$b), (fadd node:$a, node:$b),[{ return N->getFlags().hasAllowContract(); }]>; +def fsub_contract : PatFrag<(ops node:$a, node:$b), (fsub node:$a, node:$b),[{ + return N->getFlags().hasAllowContract(); +}]>; def not : PatFrag<(ops node:$in), (xor node:$in, -1)>; def vnot : PatFrag<(ops node:$in), (xor node:$in, immAllOnesV)>; diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h index 59e8117..8e83b046 100644 --- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h +++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h @@ -276,14 +276,14 @@ LLVM_ABI bool isX18ReservedByDefault(const Triple &TT); // For a given set of feature names, which can be either target-features, or // fmv-features metadata, expand their dependencies and then return a bitmask // corresponding to the entries of AArch64::FeatPriorities. -LLVM_ABI uint64_t getFMVPriority(ArrayRef<StringRef> Features); +LLVM_ABI APInt getFMVPriority(ArrayRef<StringRef> Features); // For a given set of FMV feature names, expand their dependencies and then // return a bitmask corresponding to the entries of AArch64::CPUFeatures. // The values in CPUFeatures are not bitmasks themselves, they are sequential // (0, 1, 2, 3, ...). The resulting bitmask is used at runtime to test whether // a certain FMV feature is available on the host. -LLVM_ABI uint64_t getCpuSupportsMask(ArrayRef<StringRef> Features); +LLVM_ABI APInt getCpuSupportsMask(ArrayRef<StringRef> Features); LLVM_ABI void PrintSupportedExtensions(); diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h index 670a632..ede9797 100644 --- a/llvm/include/llvm/TargetParser/Triple.h +++ b/llvm/include/llvm/TargetParser/Triple.h @@ -199,7 +199,8 @@ public: SUSE, OpenEmbedded, Intel, - LastVendorType = Intel + Meta, + LastVendorType = Meta }; enum OSType { UnknownOS, @@ -307,8 +308,8 @@ public: Mlibc, PAuthTest, - - LastEnvironmentType = PAuthTest + MTIA, + LastEnvironmentType = MTIA }; enum ObjectFormatType { UnknownObjectFormat, diff --git a/llvm/include/llvm/TextAPI/SymbolSet.h b/llvm/include/llvm/TextAPI/SymbolSet.h index cd30663..42c411a 100644 --- a/llvm/include/llvm/TextAPI/SymbolSet.h +++ b/llvm/include/llvm/TextAPI/SymbolSet.h @@ -92,6 +92,7 @@ private: public: SymbolSet() = default; + LLVM_ABI ~SymbolSet(); LLVM_ABI Symbol *addGlobal(EncodeKind Kind, StringRef Name, SymbolFlags Flags, const Target &Targ); size_t size() const { return Symbols.size(); } diff --git a/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h b/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h index 20850ba..a9a370b 100644 --- a/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h +++ b/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h @@ -41,6 +41,13 @@ public: static bool isRequired() { return true; } }; +class HipStdParMathFixupPass : public PassInfoMixin<HipStdParMathFixupPass> { +public: + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); + + static bool isRequired() { return true; } +}; + } // namespace llvm #endif // LLVM_TRANSFORMS_HIPSTDPAR_HIPSTDPAR_H diff --git a/llvm/include/llvm/Transforms/ObjCARC.h b/llvm/include/llvm/Transforms/ObjCARC.h index c927513..c4b4c4f 100644 --- a/llvm/include/llvm/Transforms/ObjCARC.h +++ b/llvm/include/llvm/Transforms/ObjCARC.h @@ -35,10 +35,6 @@ struct ObjCARCContractPass : public PassInfoMixin<ObjCARCContractPass> { LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; -struct ObjCARCAPElimPass : public PassInfoMixin<ObjCARCAPElimPass> { - LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); -}; - struct ObjCARCExpandPass : public PassInfoMixin<ObjCARCExpandPass> { LLVM_ABI PreservedAnalyses run(Function &M, FunctionAnalysisManager &AM); }; diff --git a/llvm/include/llvm/Transforms/Scalar/Reassociate.h b/llvm/include/llvm/Transforms/Scalar/Reassociate.h index a5d13766..749f6ee 100644 --- a/llvm/include/llvm/Transforms/Scalar/Reassociate.h +++ b/llvm/include/llvm/Transforms/Scalar/Reassociate.h @@ -28,6 +28,7 @@ #include "llvm/IR/BasicBlock.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/Support/Compiler.h" #include <deque> namespace llvm { @@ -96,7 +97,7 @@ protected: bool MadeChange; public: - PreservedAnalyses run(Function &F, FunctionAnalysisManager &); + LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &); private: void BuildRankMap(Function &F, ReversePostOrderTraversal<Function *> &RPOT); diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h index bb79d25..3f5f427 100644 --- a/llvm/include/llvm/Transforms/Utils/Local.h +++ b/llvm/include/llvm/Transforms/Utils/Local.h @@ -325,7 +325,6 @@ LLVM_ABI void salvageDebugInfo(Instruction &I); /// Mark undef if salvaging cannot be completed. LLVM_ABI void salvageDebugInfoForDbgValues(Instruction &I, - ArrayRef<DbgVariableIntrinsic *> Insns, ArrayRef<DbgVariableRecord *> DPInsns); /// Given an instruction \p I and DIExpression \p DIExpr operating on diff --git a/llvm/include/llvm/Transforms/Utils/Mem2Reg.h b/llvm/include/llvm/Transforms/Utils/Mem2Reg.h index 76c1c2c..d0006bf 100644 --- a/llvm/include/llvm/Transforms/Utils/Mem2Reg.h +++ b/llvm/include/llvm/Transforms/Utils/Mem2Reg.h @@ -15,6 +15,7 @@ #define LLVM_TRANSFORMS_UTILS_MEM2REG_H #include "llvm/IR/PassManager.h" +#include "llvm/Support/Compiler.h" namespace llvm { @@ -22,7 +23,7 @@ class Function; class PromotePass : public PassInfoMixin<PromotePass> { public: - PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h b/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h index f288bdf..e0cdcf8 100644 --- a/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h +++ b/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h @@ -57,7 +57,6 @@ struct AllocaInfo { struct StackInfo { MapVector<AllocaInst *, AllocaInfo> AllocasToInstrument; - SmallVector<Instruction *, 4> UnrecognizedLifetimes; SmallVector<Instruction *, 8> RetVec; bool CallsReturnTwice = false; }; diff --git a/llvm/include/llvm/Transforms/Utils/ProfileVerify.h b/llvm/include/llvm/Transforms/Utils/ProfileVerify.h new file mode 100644 index 0000000..5c9c44c --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/ProfileVerify.h @@ -0,0 +1,37 @@ +//===- ProfileVerify.h - Verify profile info for testing ----------*-C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Inject profile information, as part of tests, to verify passes don't +// accidentally drop it. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TRANSFORMS_UTILS_PROFILEVERIFY_H +#define LLVM_TRANSFORMS_UTILS_PROFILEVERIFY_H + +#include "llvm/IR/Analysis.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Support/Compiler.h" + +namespace llvm { +/// Inject MD_prof metadata where it's missing. Used for testing that passes +/// don't accidentally drop this metadata. +class ProfileInjectorPass : public PassInfoMixin<ProfileInjectorPass> { +public: + LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); +}; + +/// Checks that MD_prof is present on every instruction that supports it. Used +/// in conjunction with the ProfileInjectorPass. MD_prof "unknown" is considered +/// valid (i.e. !{!"unknown"}) +class ProfileVerifierPass : public PassInfoMixin<ProfileVerifierPass> { +public: + LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); +}; + +} // namespace llvm +#endif diff --git a/llvm/include/module.modulemap b/llvm/include/module.modulemap index a4aae08..ac360b2 100644 --- a/llvm/include/module.modulemap +++ b/llvm/include/module.modulemap @@ -78,6 +78,7 @@ module LLVM_BinaryFormat { textual header "llvm/BinaryFormat/DynamicTags.def" textual header "llvm/BinaryFormat/MachO.def" textual header "llvm/BinaryFormat/MinidumpConstants.def" + textual header "llvm/BinaryFormat/SFrameConstants.def" textual header "llvm/BinaryFormat/Swift.def" textual header "llvm/BinaryFormat/ELFRelocs/AArch64.def" textual header "llvm/BinaryFormat/ELFRelocs/AMDGPU.def" |