diff options
Diffstat (limited to 'llvm/include')
61 files changed, 1594 insertions, 369 deletions
diff --git a/llvm/include/llvm/ADT/AddressRanges.h b/llvm/include/llvm/ADT/AddressRanges.h index 79ba5d5..6ea097d 100644 --- a/llvm/include/llvm/ADT/AddressRanges.h +++ b/llvm/include/llvm/ADT/AddressRanges.h @@ -21,7 +21,7 @@ namespace llvm { /// a start and an end address: [Start, End). class AddressRange { public: - AddressRange() {} + AddressRange() = default; AddressRange(uint64_t S, uint64_t E) : Start(S), End(E) { assert(Start <= End); } diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h index 448d100..450f4d0 100644 --- a/llvm/include/llvm/ADT/ArrayRef.h +++ b/llvm/include/llvm/ADT/ArrayRef.h @@ -66,10 +66,6 @@ namespace llvm { /// Construct an empty ArrayRef. /*implicit*/ ArrayRef() = default; - /// Construct an empty ArrayRef from std::nullopt. - /*implicit*/ LLVM_DEPRECATED("Use {} or ArrayRef<T>() instead", "{}") - ArrayRef(std::nullopt_t) {} - /// Construct an ArrayRef from a single element. /*implicit*/ ArrayRef(const T &OneElt LLVM_LIFETIME_BOUND) : Data(&OneElt), Length(1) {} diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h index 4039078..00f85ca 100644 --- a/llvm/include/llvm/ADT/GenericCycleImpl.h +++ b/llvm/include/llvm/ADT/GenericCycleImpl.h @@ -561,6 +561,17 @@ auto GenericCycleInfo<ContextT>::getSmallestCommonCycle(CycleT *A, return A; } +/// \brief Find the innermost cycle containing both given blocks. +/// +/// \returns the innermost cycle containing both \p A and \p B +/// or nullptr if there is no such cycle. +template <typename ContextT> +auto GenericCycleInfo<ContextT>::getSmallestCommonCycle(BlockT *A, + BlockT *B) const + -> CycleT * { + return getSmallestCommonCycle(getCycle(A), getCycle(B)); +} + /// \brief get the depth for the cycle which containing a given block. /// /// \returns the depth for the innermost cycle containing \p Block or 0 if it is diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h index b8b6e3e..c31bab3 100644 --- a/llvm/include/llvm/ADT/GenericCycleInfo.h +++ b/llvm/include/llvm/ADT/GenericCycleInfo.h @@ -298,6 +298,7 @@ public: CycleT *getCycle(const BlockT *Block) const; CycleT *getSmallestCommonCycle(CycleT *A, CycleT *B) const; + CycleT *getSmallestCommonCycle(BlockT *A, BlockT *B) const; unsigned getCycleDepth(const BlockT *Block) const; CycleT *getTopLevelParentCycle(BlockT *Block); diff --git a/llvm/include/llvm/ADT/StringMap.h b/llvm/include/llvm/ADT/StringMap.h index 01cbf2d3..7901365 100644 --- a/llvm/include/llvm/ADT/StringMap.h +++ b/llvm/include/llvm/ADT/StringMap.h @@ -302,7 +302,7 @@ public: if (FindInRHS == RHS.end()) return false; - if constexpr (!std::is_same_v<ValueTy, std::nullopt_t>) { + if constexpr (!std::is_same_v<ValueTy, EmptyStringSetTag>) { if (!(KeyValue.getValue() == FindInRHS->getValue())) return false; } diff --git a/llvm/include/llvm/ADT/StringMapEntry.h b/llvm/include/llvm/ADT/StringMapEntry.h index 21be5ec..b0a3c8c 100644 --- a/llvm/include/llvm/ADT/StringMapEntry.h +++ b/llvm/include/llvm/ADT/StringMapEntry.h @@ -21,6 +21,9 @@ namespace llvm { +/// The "value type" of StringSet represented as an empty struct. +struct EmptyStringSetTag {}; + /// StringMapEntryBase - Shared base class of StringMapEntry instances. class StringMapEntryBase { size_t keyLength; @@ -85,14 +88,13 @@ public: }; template <> -class StringMapEntryStorage<std::nullopt_t> : public StringMapEntryBase { +class StringMapEntryStorage<EmptyStringSetTag> : public StringMapEntryBase { public: - explicit StringMapEntryStorage(size_t keyLength, - std::nullopt_t = std::nullopt) + explicit StringMapEntryStorage(size_t keyLength, EmptyStringSetTag = {}) : StringMapEntryBase(keyLength) {} StringMapEntryStorage(StringMapEntryStorage &entry) = delete; - std::nullopt_t getValue() const { return std::nullopt; } + EmptyStringSetTag getValue() const { return {}; } }; /// StringMapEntry - This is used to represent one value that is inserted into diff --git a/llvm/include/llvm/ADT/StringSet.h b/llvm/include/llvm/ADT/StringSet.h index c8be3f2..dc154af 100644 --- a/llvm/include/llvm/ADT/StringSet.h +++ b/llvm/include/llvm/ADT/StringSet.h @@ -22,8 +22,8 @@ namespace llvm { /// StringSet - A wrapper for StringMap that provides set-like functionality. template <class AllocatorTy = MallocAllocator> -class StringSet : public StringMap<std::nullopt_t, AllocatorTy> { - using Base = StringMap<std::nullopt_t, AllocatorTy>; +class StringSet : public StringMap<EmptyStringSetTag, AllocatorTy> { + using Base = StringMap<EmptyStringSetTag, AllocatorTy>; public: StringSet() = default; diff --git a/llvm/include/llvm/ADT/StringSwitch.h b/llvm/include/llvm/ADT/StringSwitch.h index 98685de..4868153 100644 --- a/llvm/include/llvm/ADT/StringSwitch.h +++ b/llvm/include/llvm/ADT/StringSwitch.h @@ -173,6 +173,7 @@ public: return CasesLowerImpl(CaseStrings, Value); } + [[deprecated("Pass cases in std::initializer_list instead")]] StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, T Value) { return CasesLowerImpl({S0, S1}, Value); } diff --git a/llvm/include/llvm/ADT/TypeSwitch.h b/llvm/include/llvm/ADT/TypeSwitch.h index 5657303..50ca1d5 100644 --- a/llvm/include/llvm/ADT/TypeSwitch.h +++ b/llvm/include/llvm/ADT/TypeSwitch.h @@ -111,6 +111,7 @@ public: return std::move(*result); return defaultFn(this->value); } + /// As a default, return the given value. [[nodiscard]] ResultT Default(ResultT defaultResult) { if (result) @@ -118,6 +119,22 @@ public: return defaultResult; } + /// Default for pointer-like results types that accept `nullptr`. + template <typename ArgT = ResultT, + typename = + std::enable_if_t<std::is_constructible_v<ArgT, std::nullptr_t>>> + [[nodiscard]] ResultT Default(std::nullptr_t) { + return Default(ResultT(nullptr)); + } + + /// Default for optional results types that accept `std::nullopt`. + template <typename ArgT = ResultT, + typename = + std::enable_if_t<std::is_constructible_v<ArgT, std::nullopt_t>>> + [[nodiscard]] ResultT Default(std::nullopt_t) { + return Default(ResultT(std::nullopt)); + } + /// Declare default as unreachable, making sure that all cases were handled. [[nodiscard]] ResultT DefaultUnreachable( const char *message = "Fell off the end of a type-switch") { diff --git a/llvm/include/llvm/Analysis/IR2Vec.h b/llvm/include/llvm/Analysis/IR2Vec.h index 71055dd16..e3a0b3f 100644 --- a/llvm/include/llvm/Analysis/IR2Vec.h +++ b/llvm/include/llvm/Analysis/IR2Vec.h @@ -72,7 +72,7 @@ enum class IR2VecKind { Symbolic, FlowAware }; namespace ir2vec { -extern llvm::cl::OptionCategory IR2VecCategory; +LLVM_ABI extern llvm::cl::OptionCategory IR2VecCategory; LLVM_ABI extern cl::opt<float> OpcWeight; LLVM_ABI extern cl::opt<float> TypeWeight; LLVM_ABI extern cl::opt<float> ArgWeight; diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index 6ee6b666..39e9611 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -1125,6 +1125,8 @@ struct Elf64_Shdr { Elf64_Xword sh_entsize; }; +enum { PN_XNUM = 0xffff }; + // Special section indices. enum { SHN_UNDEF = 0, // Undefined, missing, irrelevant, or meaningless diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h index 48650a6..8237530 100644 --- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h +++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h @@ -54,6 +54,10 @@ struct FunctionPathAndClusterInfo { DenseMap<UniqueBBID, uint64_t> NodeCounts; // Edge counts for each edge, stored as a nested map. DenseMap<UniqueBBID, DenseMap<UniqueBBID, uint64_t>> EdgeCounts; + // Hash for each basic block. The Hashes are stored for every original block + // (not cloned blocks), hence the map key being unsigned instead of + // UniqueBBID. + DenseMap<unsigned, uint64_t> BBHashes; }; class BasicBlockSectionsProfileReader { diff --git a/llvm/include/llvm/CodeGen/MIR2Vec.h b/llvm/include/llvm/CodeGen/MIR2Vec.h index 44f009c..18b1290 100644 --- a/llvm/include/llvm/CodeGen/MIR2Vec.h +++ b/llvm/include/llvm/CodeGen/MIR2Vec.h @@ -73,7 +73,7 @@ namespace mir2vec { class MIREmbedder; class SymbolicMIREmbedder; -extern llvm::cl::OptionCategory MIR2VecCategory; +LLVM_ABI extern llvm::cl::OptionCategory MIR2VecCategory; extern cl::opt<float> OpcWeight, CommonOperandWeight, RegOperandWeight; using Embedding = ir2vec::Embedding; @@ -154,14 +154,14 @@ class MIRVocabulary { void buildRegisterOperandMapping(); /// Get canonical index for a machine opcode - unsigned getCanonicalOpcodeIndex(unsigned Opcode) const; + LLVM_ABI unsigned getCanonicalOpcodeIndex(unsigned Opcode) const; /// Get index for a common (non-register) machine operand unsigned getCommonOperandIndex(MachineOperand::MachineOperandType OperandType) const; /// Get index for a register machine operand - unsigned getRegisterOperandIndex(Register Reg) const; + LLVM_ABI unsigned getRegisterOperandIndex(Register Reg) const; // Accessors for operand types const Embedding & @@ -192,7 +192,7 @@ class MIRVocabulary { /// Get entity ID (flat index) for a common operand type /// This is used for triplet generation - unsigned getEntityIDForCommonOperand( + LLVM_ABI unsigned getEntityIDForCommonOperand( MachineOperand::MachineOperandType OperandType) const { return Layout.CommonOperandBase + getCommonOperandIndex(OperandType); } @@ -221,7 +221,7 @@ public: bool IsPhysical = true) const; /// Get the string key for a vocabulary entry at the given position - std::string getStringKey(unsigned Pos) const; + LLVM_ABI std::string getStringKey(unsigned Pos) const; unsigned getDimension() const { return Storage.getDimension(); } @@ -268,7 +268,7 @@ public: const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI); /// Create a dummy vocabulary for testing purposes. - static Expected<MIRVocabulary> + LLVM_ABI static Expected<MIRVocabulary> createDummyVocabForTest(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, unsigned Dim = 1); @@ -302,10 +302,10 @@ protected: RegOperandWeight(mir2vec::RegOperandWeight) {} /// Function to compute embeddings. - Embedding computeEmbeddings() const; + LLVM_ABI Embedding computeEmbeddings() const; /// Function to compute the embedding for a given machine basic block. - Embedding computeEmbeddings(const MachineBasicBlock &MBB) const; + LLVM_ABI Embedding computeEmbeddings(const MachineBasicBlock &MBB) const; /// Function to compute the embedding for a given machine instruction. /// Specific to the kind of embeddings being computed. @@ -316,9 +316,9 @@ public: /// Factory method to create an Embedder object of the specified kind /// Returns nullptr if the requested kind is not supported. - static std::unique_ptr<MIREmbedder> create(MIR2VecKind Mode, - const MachineFunction &MF, - const MIRVocabulary &Vocab); + LLVM_ABI static std::unique_ptr<MIREmbedder> + create(MIR2VecKind Mode, const MachineFunction &MF, + const MIRVocabulary &Vocab); /// Computes and returns the embedding for a given machine instruction MI in /// the machine function MF. @@ -369,7 +369,7 @@ class MIR2VecVocabProvider { public: MIR2VecVocabProvider(const MachineModuleInfo &MMI) : MMI(MMI) {} - Expected<mir2vec::MIRVocabulary> getVocabulary(const Module &M); + LLVM_ABI Expected<mir2vec::MIRVocabulary> getVocabulary(const Module &M); private: Error readVocabulary(VocabMap &OpcVocab, VocabMap &CommonOperandVocab, @@ -454,7 +454,7 @@ public: }; /// Create a machine pass that prints MIR2Vec embeddings -MachineFunctionPass *createMIR2VecPrinterLegacyPass(raw_ostream &OS); +LLVM_ABI MachineFunctionPass *createMIR2VecPrinterLegacyPass(raw_ostream &OS); } // namespace llvm diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index df6ce0f..1a5ffb3 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1113,7 +1113,8 @@ public: SDValue Mask, SDValue EVL); /// Returns sum of the base pointer and offset. - /// Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default. + /// Unlike getObjectPtrOffset this does not set NoUnsignedWrap and InBounds by + /// default. LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags = SDNodeFlags()); @@ -1123,15 +1124,18 @@ public: /// Create an add instruction with appropriate flags when used for /// addressing some offset of an object. i.e. if a load is split into multiple - /// components, create an add nuw from the base pointer to the offset. + /// components, create an add nuw (or ptradd nuw inbounds) from the base + /// pointer to the offset. SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset) { - return getMemBasePlusOffset(Ptr, Offset, SL, SDNodeFlags::NoUnsignedWrap); + return getMemBasePlusOffset( + Ptr, Offset, SL, SDNodeFlags::NoUnsignedWrap | SDNodeFlags::InBounds); } SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, SDValue Offset) { // The object itself can't wrap around the address space, so it shouldn't be // possible for the adds of the offsets to the split parts to overflow. - return getMemBasePlusOffset(Ptr, Offset, SL, SDNodeFlags::NoUnsignedWrap); + return getMemBasePlusOffset( + Ptr, Offset, SL, SDNodeFlags::NoUnsignedWrap | SDNodeFlags::InBounds); } /// Return a new CALLSEQ_START node, that starts new call frame, in which diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 1920b98..78f63b4 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -5649,17 +5649,35 @@ public: /// Get a pointer to vector element \p Idx located in memory for a vector of /// type \p VecVT starting at a base address of \p VecPtr. If \p Idx is out of /// bounds the returned pointer is unspecified, but will be within the vector - /// bounds. - SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, - SDValue Index) const; + /// bounds. \p PtrArithFlags can be used to mark that arithmetic within the + /// vector in memory is known to not wrap or to be inbounds. + SDValue getVectorElementPointer( + SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index, + const SDNodeFlags PtrArithFlags = SDNodeFlags()) const; + + /// Get a pointer to vector element \p Idx located in memory for a vector of + /// type \p VecVT starting at a base address of \p VecPtr. If \p Idx is out of + /// bounds the returned pointer is unspecified, but will be within the vector + /// bounds. \p VecPtr is guaranteed to point to the beginning of a memory + /// location large enough for the vector. + SDValue getInboundsVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, + EVT VecVT, SDValue Index) const { + return getVectorElementPointer(DAG, VecPtr, VecVT, Index, + SDNodeFlags::NoUnsignedWrap | + SDNodeFlags::InBounds); + } /// Get a pointer to a sub-vector of type \p SubVecVT at index \p Idx located /// in memory for a vector of type \p VecVT starting at a base address of /// \p VecPtr. If \p Idx plus the size of \p SubVecVT is out of bounds the /// returned pointer is unspecified, but the value returned will be such that - /// the entire subvector would be within the vector bounds. - SDValue getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, - EVT SubVecVT, SDValue Index) const; + /// the entire subvector would be within the vector bounds. \p PtrArithFlags + /// can be used to mark that arithmetic within the vector in memory is known + /// to not wrap or to be inbounds. + SDValue + getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, + EVT SubVecVT, SDValue Index, + const SDNodeFlags PtrArithFlags = SDNodeFlags()) const; /// Method for building the DAG expansion of ISD::[US][MIN|MAX]. This /// method accepts integers as its arguments. diff --git a/llvm/include/llvm/CodeGen/TileShapeInfo.h b/llvm/include/llvm/CodeGen/TileShapeInfo.h index 9cea327..24d9de8 100644 --- a/llvm/include/llvm/CodeGen/TileShapeInfo.h +++ b/llvm/include/llvm/CodeGen/TileShapeInfo.h @@ -34,30 +34,9 @@ public: if (MRI) deduceImm(MRI); } - // When ShapeT has multiple shapes, we only use Shapes (never use Row and Col) - // and ImmShapes. Due to the most case is only one shape (just simply use - // Shape.Row or Shape.Col), so here we don't merge Row and Col into vector - // Shapes to keep the speed and code simplicity. - // TODO: The upper solution is a temporary way to minimize current tile - // register allocation code changes. It can not handle both Reg shape and - // Imm shape for different shapes (e.g. shape 1 is reg shape while shape 2 - // is imm shape). Refine me when we have more multi-tile shape instructions! - ShapeT(ArrayRef<MachineOperand *> ShapesOperands, - const MachineRegisterInfo *MRI = nullptr) - : Row(nullptr), Col(nullptr), RowImm(InvalidImmShape), - ColImm(InvalidImmShape) { - assert(ShapesOperands.size() % 2 == 0 && "Miss row or col!"); - - llvm::append_range(Shapes, ShapesOperands); - - if (MRI) - deduceImm(MRI); - } ShapeT() : Row(nullptr), Col(nullptr), RowImm(InvalidImmShape), ColImm(InvalidImmShape) {} - // TODO: We need to extern cmp operator for multi-shapes if - // we have requirement in the future. bool operator==(const ShapeT &Shape) const { MachineOperand *R = Shape.Row; MachineOperand *C = Shape.Col; @@ -74,40 +53,11 @@ public: bool operator!=(const ShapeT &Shape) const { return !(*this == Shape); } - MachineOperand *getRow(unsigned I = 0) const { - if (Shapes.empty()) - return Row; - assert(Shapes.size() / 2 >= I && "Get invalid row from id!"); - return Shapes[I * 2]; - } - - MachineOperand *getCol(unsigned I = 0) const { - if (Shapes.empty()) - return Col; - assert(Shapes.size() / 2 >= I && "Get invalid col from id!"); - return Shapes[I * 2 + 1]; - } - - int64_t getRowImm(unsigned I = 0) const { - if (ImmShapes.empty()) - return RowImm; - assert(ImmShapes.size() / 2 >= I && "Get invalid imm row from id!"); - return ImmShapes[I * 2]; - } - - int64_t getColImm(unsigned I = 0) const { - if (ImmShapes.empty()) - return ColImm; - assert(ImmShapes.size() / 2 >= I && "Get invalid imm col from id!"); - return ImmShapes[I * 2 + 1]; - } + MachineOperand *getRow() const { return Row; } + MachineOperand *getCol() const { return Col; } - unsigned getShapeNum() { - if (Shapes.empty()) - return isValid() ? 1 : 0; - else - return Shapes.size() / 2; - } + int64_t getRowImm() const { return RowImm; } + int64_t getColImm() const { return ColImm; } bool isValid() { return (Row != nullptr) && (Col != nullptr); } @@ -120,35 +70,14 @@ public: for (const MachineOperand &DefMO : MRI->def_operands(Reg)) { const auto *MI = DefMO.getParent(); if (MI->isMoveImmediate()) { - assert(MI->getNumOperands() == 2 && - "Unsupported number of operands in instruction for setting " - "row/column."); - if (MI->getOperand(1).isImm()) { - Imm = MI->getOperand(1).getImm(); - } else { - assert(MI->getOperand(1).isImplicit() && - "Operand 1 is assumed to be implicit."); - Imm = 0; - } + Imm = MI->getOperand(1).getImm(); break; } } return Imm; }; - if (Shapes.empty()) { // Single Shape - RowImm = GetImm(Row->getReg()); - ColImm = GetImm(Col->getReg()); - // The number of rows of 2nd destination buffer is assigned by the one of - // 1st destination buffer. If the column size is equal to zero, the row - // size should be reset to zero too. - if (ColImm == 0) - Row = Col; - } else { // Multiple Shapes - for (auto *Shape : Shapes) { - int64_t ImmShape = GetImm(Shape->getReg()); - ImmShapes.push_back(ImmShape); - } - } + RowImm = GetImm(Row->getReg()); + ColImm = GetImm(Col->getReg()); } private: @@ -157,9 +86,6 @@ private: MachineOperand *Col; int64_t RowImm = -1; int64_t ColImm = -1; - // Multiple Shapes - SmallVector<MachineOperand *, 0> Shapes; - SmallVector<int64_t, 0> ImmShapes; }; } // namespace llvm diff --git a/llvm/include/llvm/CodeGenTypes/LowLevelType.h b/llvm/include/llvm/CodeGenTypes/LowLevelType.h index 4c1fe13..472a3f3 100644 --- a/llvm/include/llvm/CodeGenTypes/LowLevelType.h +++ b/llvm/include/llvm/CodeGenTypes/LowLevelType.h @@ -340,18 +340,18 @@ private: /// valid encodings, SizeInBits/SizeOfElement must be larger than 0. /// * Non-pointer scalar (isPointer == 0 && isVector == 0): /// SizeInBits: 32; - static const constexpr BitFieldInfo ScalarSizeFieldInfo{32, 29}; + static constexpr BitFieldInfo ScalarSizeFieldInfo{32, 29}; /// * Pointer (isPointer == 1 && isVector == 0): /// SizeInBits: 16; /// AddressSpace: 24; - static const constexpr BitFieldInfo PointerSizeFieldInfo{16, 45}; - static const constexpr BitFieldInfo PointerAddressSpaceFieldInfo{24, 21}; + static constexpr BitFieldInfo PointerSizeFieldInfo{16, 45}; + static constexpr BitFieldInfo PointerAddressSpaceFieldInfo{24, 21}; /// * Vector-of-non-pointer (isPointer == 0 && isVector == 1): /// NumElements: 16; /// SizeOfElement: 32; /// Scalable: 1; - static const constexpr BitFieldInfo VectorElementsFieldInfo{16, 5}; - static const constexpr BitFieldInfo VectorScalableFieldInfo{1, 0}; + static constexpr BitFieldInfo VectorElementsFieldInfo{16, 5}; + static constexpr BitFieldInfo VectorScalableFieldInfo{1, 0}; /// * Vector-of-pointer (isPointer == 1 && isVector == 1): /// NumElements: 16; /// SizeOfElement: 16; diff --git a/llvm/include/llvm/DWARFLinker/StringPool.h b/llvm/include/llvm/DWARFLinker/StringPool.h index d0f4e21..7838e3b 100644 --- a/llvm/include/llvm/DWARFLinker/StringPool.h +++ b/llvm/include/llvm/DWARFLinker/StringPool.h @@ -20,7 +20,7 @@ namespace dwarf_linker { /// StringEntry keeps data of the string: the length, external offset /// and a string body which is placed right after StringEntry. -using StringEntry = StringMapEntry<std::nullopt_t>; +using StringEntry = StringMapEntry<EmptyStringSetTag>; class StringPoolEntryInfo { public: diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h index c69b6f7..8620726 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// Implements ExecutorProcessControl::MemoryAccess by making calls to +// Implements the MemoryAccess interface by making calls to // ExecutorProcessControl::callWrapperAsync. // // This simplifies the implementaton of new ExecutorProcessControl instances, @@ -19,6 +19,7 @@ #define LLVM_EXECUTIONENGINE_ORC_EPCGENERICMEMORYACCESS_H #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/MemoryAccess.h" namespace llvm { namespace orc { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/SymbolFilter.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/SymbolFilter.h new file mode 100644 index 0000000..5170893 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/SymbolFilter.h @@ -0,0 +1,173 @@ +//===- SymbolFilter.h - Utilities for Symbol Filtering ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_SYMBOLFILTER_H +#define LLVM_EXECUTIONENGINE_ORC_SHARED_SYMBOLFILTER_H + +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" + +#include <cmath> +#include <type_traits> +#include <vector> + +namespace llvm { +namespace orc { + +namespace shared { +using SPSBloomFilter = + SPSTuple<bool, uint32_t, uint32_t, uint32_t, SPSSequence<uint64_t>>; +} + +class BloomFilter { +public: + using HashFunc = std::function<uint32_t(StringRef)>; + + BloomFilter() = default; + BloomFilter(BloomFilter &&) noexcept = default; + BloomFilter &operator=(BloomFilter &&) noexcept = default; + BloomFilter(const BloomFilter &) = delete; + BloomFilter &operator=(const BloomFilter &) = delete; + + BloomFilter(uint32_t SymbolCount, float FalsePositiveRate, HashFunc hashFn) + : HashFn(std::move(hashFn)) { + initialize(SymbolCount, FalsePositiveRate); + } + bool isInitialized() const { return Initialized; } + + void add(StringRef Sym) { + assert(Initialized); + addHash(HashFn(Sym)); + } + + bool mayContain(StringRef Sym) const { + return !isEmpty() && testHash(HashFn(Sym)); + } + + bool isEmpty() const { return SymbolCount == 0; } + +private: + friend class shared::SPSSerializationTraits<shared::SPSBloomFilter, + BloomFilter>; + static constexpr uint32_t BitsPerEntry = 64; + + bool Initialized = false; + uint32_t SymbolCount = 0; + uint32_t BloomSize = 0; + uint32_t BloomShift = 0; + std::vector<uint64_t> BloomTable; + HashFunc HashFn; + + void initialize(uint32_t SymCount, float FalsePositiveRate) { + assert(SymCount > 0); + SymbolCount = SymCount; + Initialized = true; + + float ln2 = std::log(2.0f); + float M = -1.0f * SymbolCount * std::log(FalsePositiveRate) / (ln2 * ln2); + BloomSize = static_cast<uint32_t>(std::ceil(M / BitsPerEntry)); + BloomShift = std::min(6u, log2ceil(SymbolCount)); + BloomTable.resize(BloomSize, 0); + } + + void addHash(uint32_t Hash) { + uint32_t Hash2 = Hash >> BloomShift; + uint32_t N = (Hash / BitsPerEntry) % BloomSize; + uint64_t Mask = + (1ULL << (Hash % BitsPerEntry)) | (1ULL << (Hash2 % BitsPerEntry)); + BloomTable[N] |= Mask; + } + + bool testHash(uint32_t Hash) const { + uint32_t Hash2 = Hash >> BloomShift; + uint32_t N = (Hash / BitsPerEntry) % BloomSize; + uint64_t Mask = + (1ULL << (Hash % BitsPerEntry)) | (1ULL << (Hash2 % BitsPerEntry)); + return (BloomTable[N] & Mask) == Mask; + } + + static constexpr uint32_t log2ceil(uint32_t V) { + return V <= 1 ? 0 : 32 - countl_zero(V - 1); + } +}; + +class BloomFilterBuilder { +public: + using HashFunc = BloomFilter::HashFunc; + + BloomFilterBuilder() = default; + + BloomFilterBuilder &setFalsePositiveRate(float Rate) { + assert(Rate > 0.0f && Rate < 1.0f); + FalsePositiveRate = Rate; + return *this; + } + + BloomFilterBuilder &setHashFunction(HashFunc Fn) { + HashFn = std::move(Fn); + return *this; + } + + BloomFilter build(ArrayRef<StringRef> Symbols) const { + assert(!Symbols.empty() && "Cannot build filter from empty symbol list."); + BloomFilter F(static_cast<uint32_t>(Symbols.size()), FalsePositiveRate, + HashFn); + for (const auto &Sym : Symbols) + F.add(Sym); + + return F; + } + +private: + float FalsePositiveRate = 0.02f; + HashFunc HashFn = [](StringRef S) -> uint32_t { + uint32_t H = 5381; + for (char C : S) + H = ((H << 5) + H) + static_cast<uint8_t>(C); // H * 33 + C + return H; + }; +}; + +namespace shared { + +template <> class SPSSerializationTraits<SPSBloomFilter, BloomFilter> { +public: + static size_t size(const BloomFilter &Filter) { + return SPSBloomFilter::AsArgList::size( + Filter.Initialized, Filter.SymbolCount, Filter.BloomSize, + Filter.BloomShift, Filter.BloomTable); + } + + static bool serialize(SPSOutputBuffer &OB, const BloomFilter &Filter) { + return SPSBloomFilter::AsArgList::serialize( + OB, Filter.Initialized, Filter.SymbolCount, Filter.BloomSize, + Filter.BloomShift, Filter.BloomTable); + } + + static bool deserialize(SPSInputBuffer &IB, BloomFilter &Filter) { + bool IsInitialized; + uint32_t SymbolCount = 0, BloomSize = 0, BloomShift = 0; + std::vector<uint64_t> BloomTable; + + if (!SPSBloomFilter::AsArgList::deserialize( + IB, IsInitialized, SymbolCount, BloomSize, BloomShift, BloomTable)) + return false; + + Filter.Initialized = IsInitialized; + Filter.SymbolCount = SymbolCount; + Filter.BloomSize = BloomSize; + Filter.BloomShift = BloomShift; + Filter.BloomTable = std::move(BloomTable); + + return true; + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm +#endif // LLVM_EXECUTIONENGINE_ORC_SHARED_SYMBOLFILTER_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/LibraryResolver.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/LibraryResolver.h new file mode 100644 index 0000000..9182995 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/LibraryResolver.h @@ -0,0 +1,511 @@ +//===- LibraryResolver.h - Automatic Library Symbol Resolution -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides support for automatically searching symbols across +// dynamic libraries that have not yet been loaded. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_LIBRARYRESOLVER_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_LIBRARYRESOLVER_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ExecutionEngine/Orc/Shared/SymbolFilter.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/LibraryScanner.h" +#include "llvm/Support/Path.h" + +#include <atomic> +#include <shared_mutex> +#include <unordered_map> + +namespace llvm { +namespace orc { + +/// Manages library metadata and state for symbol resolution. +/// +/// Tracks libraries by load state and kind (user/system), and stores +/// associated Bloom filters and hash maps to speed up symbol lookups. +/// Thread-safe for concurrent access. +class LibraryManager { +public: + enum class LibState : uint8_t { Unloaded = 0, Loaded = 1, Queried = 2 }; + + class LibraryInfo { + public: + LibraryInfo(const LibraryInfo &) = delete; + LibraryInfo &operator=(const LibraryInfo &) = delete; + + LibraryInfo(std::string FilePath, LibState S, PathType K, + std::optional<BloomFilter> Filter = std::nullopt) + : FilePath(std::move(FilePath)), S(S), K(K), Filter(std::move(Filter)) { + } + + StringRef getBasePath() const { return sys::path::parent_path(FilePath); } + StringRef getFileName() const { return sys::path::filename(FilePath); } + + std::string getFullPath() const { return FilePath; } + + void setFilter(BloomFilter F) { + std::lock_guard<std::shared_mutex> Lock(Mtx); + if (Filter) + return; + Filter.emplace(std::move(F)); + } + + void ensureFilterBuilt(const BloomFilterBuilder &FB, + ArrayRef<StringRef> Symbols) { + std::lock_guard<std::shared_mutex> Lock(Mtx); + if (Filter) + return; + Filter.emplace(FB.build(Symbols)); + } + + bool mayContain(StringRef Symbol) const { + assert(hasFilter()); + std::shared_lock<std::shared_mutex> Lock(Mtx); + return Filter->mayContain(Symbol); + } + + bool hasFilter() const { + std::shared_lock<std::shared_mutex> Lock(Mtx); + return Filter.has_value(); + } + + LibState getState() const { return S.load(); } + PathType getKind() const { return K; } + + void setState(LibState s) { S.store(s); } + + bool operator==(const LibraryInfo &other) const { + return FilePath == other.FilePath; + } + + private: + std::string FilePath; + std::atomic<LibState> S; + PathType K; + std::optional<BloomFilter> Filter; + mutable std::shared_mutex Mtx; + }; + + /// A read-only view of libraries filtered by state and kind. + /// + /// Lets you loop over only the libraries in a map that match a given State + /// and PathType. + class FilteredView { + public: + using Map = StringMap<std::shared_ptr<LibraryInfo>>; + using Iterator = typename Map::const_iterator; + class FilterIterator { + public: + FilterIterator(Iterator it_, Iterator end_, LibState S, PathType K) + : it(it_), end(end_), S(S), K(K) { + advance(); + } + + bool operator!=(const FilterIterator &other) const { + return it != other.it; + } + + const std::shared_ptr<LibraryInfo> &operator*() const { + return it->second; + } + + FilterIterator &operator++() { + ++it; + advance(); + return *this; + } + + private: + void advance() { + for (; it != end; ++it) + if (it->second->getState() == S && it->second->getKind() == K) + break; + } + Iterator it; + Iterator end; + LibState S; + PathType K; + }; + FilteredView(Iterator begin, Iterator end, LibState s, PathType k) + : mapBegin(begin), mapEnd(end), state(s), kind(k) {} + + FilterIterator begin() const { + return FilterIterator(mapBegin, mapEnd, state, kind); + } + + FilterIterator end() const { + return FilterIterator(mapEnd, mapEnd, state, kind); + } + + private: + Iterator mapBegin; + Iterator mapEnd; + LibState state; + PathType kind; + }; + +private: + StringMap<std::shared_ptr<LibraryInfo>> Libraries; + mutable std::shared_mutex Mtx; + +public: + using LibraryVisitor = std::function<bool(const LibraryInfo &)>; + + LibraryManager() = default; + ~LibraryManager() = default; + + bool addLibrary(std::string Path, PathType Kind, + std::optional<BloomFilter> Filter = std::nullopt) { + std::unique_lock<std::shared_mutex> Lock(Mtx); + if (Libraries.count(Path) > 0) + return false; + Libraries.insert({std::move(Path), + std::make_shared<LibraryInfo>(Path, LibState::Unloaded, + Kind, std::move(Filter))}); + return true; + } + + bool hasLibrary(StringRef Path) const { + std::shared_lock<std::shared_mutex> Lock(Mtx); + if (Libraries.count(Path) > 0) + return true; + return false; + } + + void removeLibrary(StringRef Path) { + std::unique_lock<std::shared_mutex> Lock(Mtx); + auto I = Libraries.find(Path); + if (I == Libraries.end()) + return; + Libraries.erase(I); + } + + void markLoaded(StringRef Path) { + std::unique_lock<std::shared_mutex> Lock(Mtx); + if (auto It = Libraries.find(Path); It != Libraries.end()) + It->second->setState(LibState::Loaded); + } + + void markQueried(StringRef Path) { + std::unique_lock<std::shared_mutex> Lock(Mtx); + if (auto It = Libraries.find(Path); It != Libraries.end()) + It->second->setState(LibState::Queried); + } + + std::shared_ptr<LibraryInfo> getLibrary(StringRef Path) { + std::shared_lock<std::shared_mutex> Lock(Mtx); + if (auto It = Libraries.find(Path); It != Libraries.end()) + return It->second; + return nullptr; + } + + FilteredView getView(LibState S, PathType K) const { + std::shared_lock<std::shared_mutex> Lock(Mtx); + return FilteredView(Libraries.begin(), Libraries.end(), S, K); + } + + void forEachLibrary(const LibraryVisitor &visitor) const { + std::unique_lock<std::shared_mutex> Lock(Mtx); + for (const auto &[_, entry] : Libraries) { + if (!visitor(*entry)) + break; + } + } + + bool isLoaded(StringRef Path) const { + std::unique_lock<std::shared_mutex> Lock(Mtx); + if (auto It = Libraries.find(Path.str()); It != Libraries.end()) + return It->second->getState() == LibState::Loaded; + return false; + } + + bool isQueried(StringRef Path) const { + std::unique_lock<std::shared_mutex> Lock(Mtx); + if (auto It = Libraries.find(Path.str()); It != Libraries.end()) + return It->second->getState() == LibState::Queried; + return false; + } + + void clear() { + std::unique_lock<std::shared_mutex> Lock(Mtx); + Libraries.clear(); + } +}; + +using LibraryInfo = LibraryManager::LibraryInfo; + +struct SearchPlanEntry { + LibraryManager::LibState State; // Loaded, Queried, Unloaded + PathType Type; // User, System +}; + +struct SearchPolicy { + std::vector<SearchPlanEntry> Plan; + + static SearchPolicy defaultPlan() { + return {{{LibraryManager::LibState::Loaded, PathType::User}, + {LibraryManager::LibState::Queried, PathType::User}, + {LibraryManager::LibState::Unloaded, PathType::User}, + {LibraryManager::LibState::Loaded, PathType::System}, + {LibraryManager::LibState::Queried, PathType::System}, + {LibraryManager::LibState::Unloaded, PathType::System}}}; + } +}; + +struct SymbolEnumeratorOptions { + enum Filter : uint32_t { + None = 0, + IgnoreUndefined = 1 << 0, + IgnoreWeak = 1 << 1, + IgnoreIndirect = 1 << 2, + IgnoreHidden = 1 << 3, + IgnoreNonGlobal = 1 << 4 + }; + + static SymbolEnumeratorOptions defaultOptions() { + return {Filter::IgnoreUndefined | Filter::IgnoreWeak | + Filter::IgnoreIndirect}; + } + uint32_t FilterFlags = Filter::None; +}; + +struct SearchConfig { + SearchPolicy Policy; + SymbolEnumeratorOptions Options; + + SearchConfig() + : Policy(SearchPolicy::defaultPlan()), // default plan + Options(SymbolEnumeratorOptions::defaultOptions()) {} +}; + +/// Scans libraries and resolves Symbols across user and system paths. +/// +/// Supports symbol enumeration and filtering via SymbolEnumerator, and tracks +/// symbol resolution results through SymbolQuery. Thread-safe and uses +/// LibraryScanHelper for efficient path resolution and caching. +class LibraryResolver { + friend class LibraryResolutionDriver; + +public: + class SymbolEnumerator { + public: + enum class EnumerateResult { Continue, Stop, Error }; + + using OnEachSymbolFn = std::function<EnumerateResult(StringRef Sym)>; + + static bool enumerateSymbols(StringRef Path, OnEachSymbolFn OnEach, + const SymbolEnumeratorOptions &Opts); + }; + + /// Tracks a set of symbols and the libraries where they are resolved. + /// + /// SymbolQuery is used to keep track of which symbols have been resolved + /// to which libraries. It supports concurrent read/write access using a + /// shared mutex, allowing multiple readers or a single writer at a time. + class SymbolQuery { + public: + /// Holds the result for a single symbol. + struct Result { + std::string Name; + std::string ResolvedLibPath; + }; + + private: + mutable std::shared_mutex Mtx; + StringMap<Result> Results; + std::atomic<size_t> ResolvedCount = 0; + + public: + explicit SymbolQuery(const std::vector<std::string> &Symbols) { + for (const auto &s : Symbols) { + if (!Results.contains(s)) + Results.insert({s, Result{s, ""}}); + } + } + + SmallVector<StringRef> getUnresolvedSymbols() const { + SmallVector<StringRef> Unresolved; + std::shared_lock<std::shared_mutex> Lock(Mtx); + for (const auto &[name, res] : Results) { + if (res.ResolvedLibPath.empty()) + Unresolved.push_back(name); + } + return Unresolved; + } + + void resolve(StringRef Sym, const std::string &LibPath) { + std::unique_lock<std::shared_mutex> Lock(Mtx); + auto It = Results.find(Sym); + if (It != Results.end() && It->second.ResolvedLibPath.empty()) { + It->second.ResolvedLibPath = LibPath; + ResolvedCount.fetch_add(1, std::memory_order_relaxed); + } + } + + bool allResolved() const { + return ResolvedCount.load(std::memory_order_relaxed) == Results.size(); + } + + bool hasUnresolved() const { + return ResolvedCount.load(std::memory_order_relaxed) < Results.size(); + } + + std::optional<StringRef> getResolvedLib(StringRef Sym) const { + std::shared_lock<std::shared_mutex> Lock(Mtx); + auto It = Results.find(Sym); + if (It != Results.end() && !It->second.ResolvedLibPath.empty()) + return StringRef(It->second.ResolvedLibPath); + return std::nullopt; + } + + bool isResolved(StringRef Sym) const { + std::shared_lock<std::shared_mutex> Lock(Mtx); + auto It = Results.find(Sym.str()); + return It != Results.end() && !It->second.ResolvedLibPath.empty(); + } + + std::vector<const Result *> getAllResults() const { + std::shared_lock<std::shared_mutex> Lock(Mtx); + std::vector<const Result *> Out; + Out.reserve(Results.size()); + for (const auto &[_, res] : Results) + Out.push_back(&res); + return Out; + } + }; + + struct Setup { + std::vector<std::string> BasePaths; + std::shared_ptr<LibraryPathCache> Cache; + std::shared_ptr<PathResolver> PResolver; + + size_t ScanBatchSize = 0; + + LibraryScanner::ShouldScanFn ShouldScanCall = [](StringRef) { + return true; + }; + + BloomFilterBuilder FilterBuilder = BloomFilterBuilder(); + + static Setup + create(std::vector<std::string> BasePaths, + std::shared_ptr<LibraryPathCache> existingCache = nullptr, + std::shared_ptr<PathResolver> existingResolver = nullptr, + LibraryScanner::ShouldScanFn customShouldScan = nullptr) { + Setup S; + S.BasePaths = std::move(BasePaths); + + S.Cache = + existingCache ? existingCache : std::make_shared<LibraryPathCache>(); + + S.PResolver = existingResolver ? existingResolver + : std::make_shared<PathResolver>(S.Cache); + + if (customShouldScan) + S.ShouldScanCall = std::move(customShouldScan); + + return S; + } + }; + + LibraryResolver() = delete; + explicit LibraryResolver(const Setup &S); + ~LibraryResolver() = default; + + using OnSearchComplete = unique_function<void(SymbolQuery &)>; + + void dump() { + int i = 0; + LibMgr.forEachLibrary([&](const LibraryInfo &Lib) -> bool { + dbgs() << ++i << ". Library Path : " << Lib.getFullPath() << " -> \n\t\t:" + << " ({Type : (" + << (Lib.getKind() == PathType::User ? "User" : "System") + << ") }, { State : " + << (Lib.getState() == LibraryManager::LibState::Loaded + ? "Loaded" + : "Unloaded") + << "})\n"; + return true; + }); + } + + void searchSymbolsInLibraries(std::vector<std::string> &SymList, + OnSearchComplete OnComplete, + const SearchConfig &Config = SearchConfig()); + +private: + bool scanLibrariesIfNeeded(PathType K, size_t BatchSize = 0); + void resolveSymbolsInLibrary(LibraryInfo &Lib, SymbolQuery &Q, + const SymbolEnumeratorOptions &Opts); + bool + symbolExistsInLibrary(const LibraryInfo &Lib, StringRef Sym, + std::vector<std::string> *MatchedSymbols = nullptr); + + bool symbolExistsInLibrary(const LibraryInfo &Lib, StringRef SymName, + std::vector<std::string> *AllSymbols, + const SymbolEnumeratorOptions &Opts); + + std::shared_ptr<LibraryPathCache> LibPathCache; + std::shared_ptr<PathResolver> LibPathResolver; + LibraryScanHelper ScanHelper; + BloomFilterBuilder FB; + LibraryManager LibMgr; + LibraryScanner::ShouldScanFn ShouldScanCall; + size_t scanBatchSize; +}; + +using SymbolEnumerator = LibraryResolver::SymbolEnumerator; +using SymbolQuery = LibraryResolver::SymbolQuery; +using EnumerateResult = SymbolEnumerator::EnumerateResult; + +class LibraryResolutionDriver { +public: + static std::unique_ptr<LibraryResolutionDriver> + create(const LibraryResolver::Setup &S); + + void addScanPath(const std::string &Path, PathType Kind); + bool markLibraryLoaded(StringRef Path); + bool markLibraryUnLoaded(StringRef Path); + bool isLibraryLoaded(StringRef Path) const { + return LR->LibMgr.isLoaded(Path); + } + + void resetAll() { + LR->LibMgr.clear(); + LR->ScanHelper.resetToScan(); + LR->LibPathCache->clear(); + } + + void scanAll(size_t BatchSize = 0) { + LR->scanLibrariesIfNeeded(PathType::User, BatchSize); + LR->scanLibrariesIfNeeded(PathType::System, BatchSize); + } + + void scan(PathType PK, size_t BatchSize = 0) { + LR->scanLibrariesIfNeeded(PK, BatchSize); + } + + void resolveSymbols(std::vector<std::string> Symbols, + LibraryResolver::OnSearchComplete OnCompletion, + const SearchConfig &Config = SearchConfig()); + + ~LibraryResolutionDriver() = default; + +private: + LibraryResolutionDriver(std::unique_ptr<LibraryResolver> L) + : LR(std::move(L)) {} + + std::unique_ptr<LibraryResolver> LR; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_LIBRARYRESOLVER_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/LibraryScanner.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/LibraryScanner.h new file mode 100644 index 0000000..d1c2013 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/LibraryScanner.h @@ -0,0 +1,474 @@ +//===- LibraryScanner.h - Scanner for Shared Libraries ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides functionality for scanning dynamic (shared) libraries. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_LIBRARYSCANNER_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_LIBRARYSCANNER_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/StringSaver.h" + +#include <atomic> +#include <mutex> +#include <queue> +#include <shared_mutex> +#include <string> +#include <unordered_map> +#include <unordered_set> + +namespace llvm { +namespace orc { + +class LibraryManager; + +class LibraryPathCache { + friend class PathResolver; + +public: + LibraryPathCache() = default; + + void clear(bool isRealPathCache = false) { + std::unique_lock<std::shared_mutex> lock(Mtx); + Seen.clear(); + if (isRealPathCache) { + RealPathCache.clear(); +#ifndef _WIN32 + ReadlinkCache.clear(); + LstatCache.clear(); +#endif + } + } + + void markSeen(const std::string &CanonPath) { + std::unique_lock<std::shared_mutex> lock(Mtx); + Seen.insert(CanonPath); + } + + bool hasSeen(StringRef CanonPath) const { + std::shared_lock<std::shared_mutex> lock(Mtx); + return Seen.contains(CanonPath); + } + + bool hasSeenOrMark(StringRef CanonPath) { + std::string s = CanonPath.str(); + { + std::shared_lock<std::shared_mutex> lock(Mtx); + if (Seen.contains(s)) + return true; + } + { + std::unique_lock<std::shared_mutex> lock(Mtx); + Seen.insert(s); + } + return false; + } + +private: + mutable std::shared_mutex Mtx; + + struct PathInfo { + std::string canonicalPath; + std::error_code ErrnoCode; + }; + + void insert_realpath(StringRef Path, const PathInfo &Info) { + std::unique_lock<std::shared_mutex> lock(Mtx); + RealPathCache.insert({Path, Info}); + } + + std::optional<PathInfo> read_realpath(StringRef Path) const { + std::shared_lock<std::shared_mutex> lock(Mtx); + auto It = RealPathCache.find(Path); + if (It != RealPathCache.end()) + return It->second; + + return std::nullopt; + } + + StringSet<> Seen; + StringMap<PathInfo> RealPathCache; + +#ifndef _WIN32 + StringMap<std::string> ReadlinkCache; + StringMap<mode_t> LstatCache; + + void insert_link(StringRef Path, const std::string &s) { + std::unique_lock<std::shared_mutex> lock(Mtx); + ReadlinkCache.insert({Path, s}); + } + + std::optional<std::string> read_link(StringRef Path) const { + std::shared_lock<std::shared_mutex> lock(Mtx); + auto It = ReadlinkCache.find(Path); + if (It != ReadlinkCache.end()) + return It->second; + + return std::nullopt; + } + + void insert_lstat(StringRef Path, mode_t m) { + std::unique_lock<std::shared_mutex> lock(Mtx); + LstatCache.insert({Path, m}); + } + + std::optional<mode_t> read_lstat(StringRef Path) const { + std::shared_lock<std::shared_mutex> lock(Mtx); + auto It = LstatCache.find(Path); + if (It != LstatCache.end()) + return It->second; + + return std::nullopt; + } + +#endif +}; + +/// Resolves file system paths with optional caching of results. +/// +/// Supports lstat, readlink, and realpath operations. Can resolve paths +/// relative to a base and handle symbolic links. Caches results to reduce +/// repeated system calls when enabled. +class PathResolver { +private: + std::shared_ptr<LibraryPathCache> LibPathCache; + +public: + PathResolver(std::shared_ptr<LibraryPathCache> cache) + : LibPathCache(std::move(cache)) {} + + std::optional<std::string> resolve(StringRef Path, std::error_code &ec) { + return realpathCached(Path, ec); + } +#ifndef _WIN32 + mode_t lstatCached(StringRef Path); + std::optional<std::string> readlinkCached(StringRef Path); +#endif + std::optional<std::string> realpathCached(StringRef Path, std::error_code &ec, + StringRef base = "", + bool baseIsResolved = false, + long symloopLevel = 40); +}; + +/// Performs placeholder substitution in dynamic library paths. +/// +/// Configures known placeholders (like @loader_path) and replaces them +/// in input paths with their resolved values. +class DylibSubstitutor { +public: + void configure(StringRef loaderPath); + + std::string substitute(StringRef input) const { + for (const auto &[ph, value] : Placeholders) { + if (input.starts_with_insensitive(ph)) + return (Twine(value) + input.drop_front(ph.size())).str(); + } + return input.str(); + } + +private: + StringMap<std::string> Placeholders; +}; + +/// Validates and normalizes dynamic library paths. +/// +/// Uses a `PathResolver` to resolve paths to their canonical form and +/// checks whether they point to valid shared libraries. +class DylibPathValidator { +public: + DylibPathValidator(PathResolver &PR) : LibPathResolver(PR) {} + + static bool isSharedLibrary(StringRef Path); + + std::optional<std::string> normalize(StringRef Path) const { + std::error_code ec; + auto real = LibPathResolver.resolve(Path, ec); + if (!real || ec) + return std::nullopt; + + return real; + } + + /// Validate the given path as a shared library. + std::optional<std::string> validate(StringRef Path) const { + auto realOpt = normalize(Path); + if (!realOpt) + return std::nullopt; + + if (!isSharedLibrary(*realOpt)) + return std::nullopt; + + return realOpt; + } + +private: + PathResolver &LibPathResolver; +}; + +enum class SearchPathType { + RPath, + UsrOrSys, + RunPath, +}; + +struct SearchPathConfig { + ArrayRef<StringRef> Paths; + SearchPathType type; +}; + +class SearchPathResolver { +public: + SearchPathResolver(const SearchPathConfig &Cfg, + StringRef PlaceholderPrefix = "") + : Kind(Cfg.type), PlaceholderPrefix(PlaceholderPrefix) { + for (auto &path : Cfg.Paths) + Paths.emplace_back(path.str()); + } + + std::optional<std::string> resolve(StringRef libStem, + const DylibSubstitutor &Subst, + DylibPathValidator &Validator) const; + SearchPathType searchPathType() const { return Kind; } + +private: + std::vector<std::string> Paths; + SearchPathType Kind; + std::string PlaceholderPrefix; +}; + +class DylibResolverImpl { +public: + DylibResolverImpl(DylibSubstitutor Substitutor, DylibPathValidator &Validator, + std::vector<SearchPathResolver> Resolvers) + : Substitutor(std::move(Substitutor)), Validator(Validator), + Resolvers(std::move(Resolvers)) {} + + std::optional<std::string> resolve(StringRef Stem, + bool VariateLibStem = false) const; + +private: + std::optional<std::string> tryWithExtensions(StringRef libstem) const; + + DylibSubstitutor Substitutor; + DylibPathValidator &Validator; + std::vector<SearchPathResolver> Resolvers; +}; + +class DylibResolver { +public: + DylibResolver(DylibPathValidator &Validator) : Validator(Validator) {} + + void configure(StringRef loaderPath, + ArrayRef<SearchPathConfig> SearchPathCfg) { + DylibSubstitutor Substitutor; + Substitutor.configure(loaderPath); + + std::vector<SearchPathResolver> Resolvers; + for (const auto &cfg : SearchPathCfg) { + Resolvers.emplace_back(cfg, + cfg.type == SearchPathType::RPath ? "@rpath" : ""); + } + + impl_ = std::make_unique<DylibResolverImpl>( + std::move(Substitutor), Validator, std::move(Resolvers)); + } + + std::optional<std::string> resolve(StringRef libStem, + bool VariateLibStem = false) const { + if (!impl_) + return std::nullopt; + return impl_->resolve(libStem, VariateLibStem); + } + + static std::string resolvelinkerFlag(StringRef libStem, + StringRef loaderPath) { + DylibSubstitutor Substitutor; + Substitutor.configure(loaderPath); + return Substitutor.substitute(libStem); + } + +private: + DylibPathValidator &Validator; + std::unique_ptr<DylibResolverImpl> impl_; +}; + +enum class PathType : uint8_t { User, System, Unknown }; + +enum class ScanState : uint8_t { NotScanned, Scanning, Scanned }; + +struct LibrarySearchPath { + std::string BasePath; // Canonical base directory path + PathType Kind; // User or System + std::atomic<ScanState> State; + + LibrarySearchPath(std::string Base, PathType K) + : BasePath(std::move(Base)), Kind(K), State(ScanState::NotScanned) {} +}; + +/// Scans and tracks libraries for symbol resolution. +/// +/// Maintains a list of library paths to scan, caches scanned units, +/// and resolves paths canonically for consistent tracking. +class LibraryScanHelper { +public: + explicit LibraryScanHelper(const std::vector<std::string> &SPaths, + std::shared_ptr<LibraryPathCache> LibPathCache, + std::shared_ptr<PathResolver> LibPathResolver) + : LibPathCache(std::move(LibPathCache)), + LibPathResolver(std::move(LibPathResolver)) { + DEBUG_WITH_TYPE( + "orc", dbgs() << "LibraryScanHelper::LibraryScanHelper: base paths : " + << SPaths.size() << "\n";); + for (const auto &p : SPaths) + addBasePath(p); + } + + void + addBasePath(const std::string &P, + PathType Kind = + PathType::Unknown); // Add a canonical directory for scanning + std::vector<std::shared_ptr<LibrarySearchPath>> + getNextBatch(PathType Kind, size_t batchSize); + + bool leftToScan(PathType K) const; + void resetToScan(); + + bool isTrackedBasePath(StringRef P) const; + std::vector<std::shared_ptr<LibrarySearchPath>> getAllUnits() const; + + SmallVector<StringRef> getSearchPaths() const { + SmallVector<StringRef> SearchPaths; + for (const auto &[_, SP] : LibSearchPaths) + SearchPaths.push_back(SP->BasePath); + return SearchPaths; + } + + PathResolver &getPathResolver() const { return *LibPathResolver; } + + LibraryPathCache &getCache() const { return *LibPathCache; } + + bool hasSeenOrMark(StringRef P) const { + return LibPathCache->hasSeenOrMark(P); + } + + std::optional<std::string> resolve(StringRef P, std::error_code &ec) const { + return LibPathResolver->resolve(P.str(), ec); + } + +private: + std::string resolveCanonical(StringRef P, std::error_code &ec) const; + PathType classifyKind(StringRef P) const; + + mutable std::shared_mutex Mtx; + std::shared_ptr<LibraryPathCache> LibPathCache; + std::shared_ptr<PathResolver> LibPathResolver; + + StringMap<std::shared_ptr<LibrarySearchPath>> + LibSearchPaths; // key: canonical path + std::deque<StringRef> UnscannedUsr; + std::deque<StringRef> UnscannedSys; +}; + +/// Loads an object file and provides access to it. +/// +/// Owns the underlying `ObjectFile` and ensures it is valid. +/// Any errors encountered during construction are stored and +/// returned when attempting to access the file. +class ObjectFileLoader { +public: + /// Construct an object file loader from the given path. + explicit ObjectFileLoader(StringRef Path) { + auto ObjOrErr = loadObjectFileWithOwnership(Path); + if (ObjOrErr) + Obj = std::move(*ObjOrErr); + else { + consumeError(std::move(Err)); + Err = ObjOrErr.takeError(); + } + } + + ObjectFileLoader(const ObjectFileLoader &) = delete; + ObjectFileLoader &operator=(const ObjectFileLoader &) = delete; + + ObjectFileLoader(ObjectFileLoader &&) = default; + ObjectFileLoader &operator=(ObjectFileLoader &&) = default; + + /// Get the loaded object file, or return an error if loading failed. + Expected<object::ObjectFile &> getObjectFile() { + if (Err) + return std::move(Err); + return *Obj.getBinary(); + } + + static bool isArchitectureCompatible(const object::ObjectFile &Obj); + +private: + object::OwningBinary<object::ObjectFile> Obj; + Error Err = Error::success(); + + static Expected<object::OwningBinary<object::ObjectFile>> + loadObjectFileWithOwnership(StringRef FilePath); +}; + +/// Scans libraries, resolves dependencies, and registers them. +class LibraryScanner { +public: + using ShouldScanFn = std::function<bool(StringRef)>; + + LibraryScanner( + LibraryScanHelper &H, LibraryManager &LibMgr, + ShouldScanFn ShouldScanCall = [](StringRef path) { return true; }) + : ScanHelper(H), LibMgr(LibMgr), + ShouldScanCall(std::move(ShouldScanCall)) {} + + void scanNext(PathType Kind, size_t batchSize = 1); + + /// Dependency info for a library. + struct LibraryDepsInfo { + llvm::BumpPtrAllocator Alloc; + llvm::StringSaver Saver{Alloc}; + + SmallVector<StringRef, 2> rpath; + SmallVector<StringRef, 2> runPath; + SmallVector<StringRef, 4> deps; + bool isPIE = false; + + void addRPath(StringRef s) { rpath.push_back(Saver.save(s)); } + + void addRunPath(StringRef s) { runPath.push_back(Saver.save(s)); } + + void addDep(StringRef s) { deps.push_back(Saver.save(s)); } + }; + +private: + LibraryScanHelper &ScanHelper; + LibraryManager &LibMgr; + ShouldScanFn ShouldScanCall; + + std::optional<std::string> shouldScan(StringRef FilePath); + Expected<LibraryDepsInfo> extractDeps(StringRef FilePath); + + void handleLibrary(StringRef P, PathType K, int level = 1); + + void scanBaseDir(std::shared_ptr<LibrarySearchPath> U); +}; + +using LibraryDepsInfo = LibraryScanner::LibraryDepsInfo; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_LIBRARYSCANNER_H diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h index 87b9520..d7f0e3a3d4 100644 --- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h +++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h @@ -1167,6 +1167,14 @@ struct ThreadsT { using EmptyTrait = std::true_type; }; +// V6.0: [14.8] `threadset` clause +template <typename T, typename I, typename E> // +struct ThreadsetT { + ENUM(ThreadsetPolicy, Omp_Pool, Omp_Team); + using WrapperTrait = std::true_type; + ThreadsetPolicy v; +}; + // V5.2: [5.9.1] `to` clause template <typename T, typename I, typename E> // struct ToT { @@ -1352,9 +1360,9 @@ using WrapperClausesT = std::variant< ProcBindT<T, I, E>, ReverseOffloadT<T, I, E>, SafelenT<T, I, E>, SelfMapsT<T, I, E>, SeverityT<T, I, E>, SharedT<T, I, E>, SimdlenT<T, I, E>, SizesT<T, I, E>, PermutationT<T, I, E>, ThreadLimitT<T, I, E>, - UnifiedAddressT<T, I, E>, UnifiedSharedMemoryT<T, I, E>, UniformT<T, I, E>, - UpdateT<T, I, E>, UseDeviceAddrT<T, I, E>, UseDevicePtrT<T, I, E>, - UsesAllocatorsT<T, I, E>>; + ThreadsetT<T, I, E>, UnifiedAddressT<T, I, E>, + UnifiedSharedMemoryT<T, I, E>, UniformT<T, I, E>, UpdateT<T, I, E>, + UseDeviceAddrT<T, I, E>, UseDevicePtrT<T, I, E>, UsesAllocatorsT<T, I, E>>; template <typename T, typename I, typename E> using UnionOfAllClausesT = typename type::Union< // diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td index 61a1a05..208609f 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMP.td +++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td @@ -539,6 +539,10 @@ def OMPC_GroupPrivate : Clause<[Spelling<"groupprivate">]> { def OMPC_Threads : Clause<[Spelling<"threads">]> { let clangClass = "OMPThreadsClause"; } +def OMPC_Threadset : Clause<[Spelling<"threadset">]> { + let clangClass = "OMPThreadsetClause"; + let flangClass = "OmpThreadsetClause"; +} def OMPC_To : Clause<[Spelling<"to">]> { let clangClass = "OMPToClause"; let flangClass = "OmpToClause"; @@ -1254,6 +1258,7 @@ def OMP_Task : Directive<[Spelling<"task">]> { VersionedClause<OMPC_Final>, VersionedClause<OMPC_If>, VersionedClause<OMPC_Priority>, + VersionedClause<OMPC_Threadset, 60>, VersionedClause<OMPC_Replayable, 60>, VersionedClause<OMPC_Transparent, 60>, ]; @@ -1297,6 +1302,7 @@ def OMP_TaskLoop : Directive<[Spelling<"taskloop">]> { VersionedClause<OMPC_Final>, VersionedClause<OMPC_If>, VersionedClause<OMPC_Priority>, + VersionedClause<OMPC_Threadset, 60>, VersionedClause<OMPC_Replayable, 60>, VersionedClause<OMPC_Transparent, 60>, ]; diff --git a/llvm/include/llvm/IR/ConstantFold.h b/llvm/include/llvm/IR/ConstantFold.h index f9f2b35..01bb128 100644 --- a/llvm/include/llvm/IR/ConstantFold.h +++ b/llvm/include/llvm/IR/ConstantFold.h @@ -26,42 +26,41 @@ #include <optional> namespace llvm { - template <typename T> class ArrayRef; - class Value; - class Constant; - class Type; +template <typename T> class ArrayRef; +class Value; +class Constant; +class Type; - // Constant fold various types of instruction... - LLVM_ABI Constant * - ConstantFoldCastInstruction(unsigned opcode, ///< The opcode of the cast - Constant *V, ///< The source constant - Type *DestTy ///< The destination type - ); - LLVM_ABI Constant *ConstantFoldSelectInstruction(Constant *Cond, Constant *V1, - Constant *V2); - LLVM_ABI Constant *ConstantFoldExtractElementInstruction(Constant *Val, - Constant *Idx); - LLVM_ABI Constant *ConstantFoldInsertElementInstruction(Constant *Val, - Constant *Elt, - Constant *Idx); - LLVM_ABI Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, - Constant *V2, - ArrayRef<int> Mask); - LLVM_ABI Constant * - ConstantFoldExtractValueInstruction(Constant *Agg, ArrayRef<unsigned> Idxs); - LLVM_ABI Constant * - ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val, - ArrayRef<unsigned> Idxs); - LLVM_ABI Constant *ConstantFoldUnaryInstruction(unsigned Opcode, Constant *V); - LLVM_ABI Constant *ConstantFoldBinaryInstruction(unsigned Opcode, - Constant *V1, Constant *V2); - LLVM_ABI Constant * - ConstantFoldCompareInstruction(CmpInst::Predicate Predicate, Constant *C1, - Constant *C2); - LLVM_ABI Constant * - ConstantFoldGetElementPtr(Type *Ty, Constant *C, - std::optional<ConstantRange> InRange, - ArrayRef<Value *> Idxs); -} // End llvm namespace +// Constant fold various types of instruction... +LLVM_ABI Constant * +ConstantFoldCastInstruction(unsigned opcode, ///< The opcode of the cast + Constant *V, ///< The source constant + Type *DestTy ///< The destination type +); +LLVM_ABI Constant *ConstantFoldSelectInstruction(Constant *Cond, Constant *V1, + Constant *V2); +LLVM_ABI Constant *ConstantFoldExtractElementInstruction(Constant *Val, + Constant *Idx); +LLVM_ABI Constant *ConstantFoldInsertElementInstruction(Constant *Val, + Constant *Elt, + Constant *Idx); +LLVM_ABI Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, + Constant *V2, + ArrayRef<int> Mask); +LLVM_ABI Constant *ConstantFoldExtractValueInstruction(Constant *Agg, + ArrayRef<unsigned> Idxs); +LLVM_ABI Constant *ConstantFoldInsertValueInstruction(Constant *Agg, + Constant *Val, + ArrayRef<unsigned> Idxs); +LLVM_ABI Constant *ConstantFoldUnaryInstruction(unsigned Opcode, Constant *V); +LLVM_ABI Constant *ConstantFoldBinaryInstruction(unsigned Opcode, Constant *V1, + Constant *V2); +LLVM_ABI Constant *ConstantFoldCompareInstruction(CmpInst::Predicate Predicate, + Constant *C1, Constant *C2); +LLVM_ABI Constant * +ConstantFoldGetElementPtr(Type *Ty, Constant *C, + std::optional<ConstantRange> InRange, + ArrayRef<Value *> Idxs); +} // namespace llvm #endif diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h index 56fc749..5445820 100644 --- a/llvm/include/llvm/IR/DataLayout.h +++ b/llvm/include/llvm/IR/DataLayout.h @@ -590,7 +590,7 @@ public: /// /// This is the amount that alloca reserves for this type. For example, /// returns 12 or 16 for x86_fp80, depending on alignment. - TypeSize getTypeAllocSize(Type *Ty) const; + LLVM_ABI TypeSize getTypeAllocSize(Type *Ty) const; /// Returns the offset in bits between successive objects of the /// specified type, including alignment padding; always a multiple of 8. diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td index 719181a..2710853 100644 --- a/llvm/include/llvm/IR/IntrinsicsNVVM.td +++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td @@ -1334,15 +1334,8 @@ let TargetPrefix = "nvvm" in { // let IntrProperties = [IntrNoMem] in { foreach ftz = ["", "_ftz"] in - def int_nvvm_ex2_approx # ftz # _f : NVVMBuiltin, - DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty]>; - - def int_nvvm_ex2_approx_d : NVVMBuiltin, - DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty]>; - def int_nvvm_ex2_approx_f16 : - DefaultAttrsIntrinsic<[llvm_half_ty], [llvm_half_ty]>; - def int_nvvm_ex2_approx_f16x2 : - DefaultAttrsIntrinsic<[llvm_v2f16_ty], [llvm_v2f16_ty]>; + def int_nvvm_ex2_approx # ftz : + DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>; foreach ftz = ["", "_ftz"] in def int_nvvm_lg2_approx # ftz # _f : NVVMBuiltin, diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td index 81fbfbf..1dd23f6 100644 --- a/llvm/include/llvm/IR/IntrinsicsX86.td +++ b/llvm/include/llvm/IR/IntrinsicsX86.td @@ -5505,46 +5505,6 @@ let TargetPrefix = "x86" in { [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; - // AMX-TRANSPOSE - def int_x86_t2rpntlvwz0 : ClangBuiltin<"__builtin_ia32_t2rpntlvwz0">, - Intrinsic<[], [llvm_i8_ty, llvm_ptr_ty, llvm_i64_ty], - [ImmArg<ArgIndex<0>>]>; - def int_x86_t2rpntlvwz0t1 : ClangBuiltin<"__builtin_ia32_t2rpntlvwz0t1">, - Intrinsic<[], [llvm_i8_ty, llvm_ptr_ty, llvm_i64_ty], - [ImmArg<ArgIndex<0>>]>; - def int_x86_t2rpntlvwz1 : ClangBuiltin<"__builtin_ia32_t2rpntlvwz1">, - Intrinsic<[], [llvm_i8_ty, llvm_ptr_ty, llvm_i64_ty], - [ImmArg<ArgIndex<0>>]>; - def int_x86_t2rpntlvwz1t1 : ClangBuiltin<"__builtin_ia32_t2rpntlvwz1t1">, - Intrinsic<[], [llvm_i8_ty, llvm_ptr_ty, llvm_i64_ty], - [ImmArg<ArgIndex<0>>]>; - def int_x86_ttransposed : ClangBuiltin<"__builtin_ia32_ttransposed">, - Intrinsic<[], [llvm_i8_ty, llvm_i8_ty], - [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>; - def int_x86_ttdpbf16ps : ClangBuiltin<"__builtin_ia32_ttdpbf16ps">, - Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], - [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, - ImmArg<ArgIndex<2>>]>; - def int_x86_ttdpfp16ps : ClangBuiltin<"__builtin_ia32_ttdpfp16ps">, - Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], - [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, - ImmArg<ArgIndex<2>>]>; - def int_x86_ttcmmimfp16ps : ClangBuiltin<"__builtin_ia32_ttcmmimfp16ps">, - Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], - [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, - ImmArg<ArgIndex<2>>]>; - def int_x86_ttcmmrlfp16ps : ClangBuiltin<"__builtin_ia32_ttcmmrlfp16ps">, - Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], - [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, - ImmArg<ArgIndex<2>>]>; - def int_x86_tconjtcmmimfp16ps : ClangBuiltin<"__builtin_ia32_tconjtcmmimfp16ps">, - Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], - [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, - ImmArg<ArgIndex<2>>]>; - def int_x86_tconjtfp16 : ClangBuiltin<"__builtin_ia32_tconjtfp16">, - Intrinsic<[], [llvm_i8_ty, llvm_i8_ty], - [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>; - // AMX-MORVS, AMX-TRANSPOSE def int_x86_t2rpntlvwz0rs : ClangBuiltin<"__builtin_ia32_t2rpntlvwz0rs">, Intrinsic<[], [llvm_i8_ty, llvm_ptr_ty, llvm_i64_ty], @@ -5685,61 +5645,6 @@ let TargetPrefix = "x86" in { [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_ptr_ty, llvm_i64_ty], [IntrArgMemOnly]>; - def int_x86_t2rpntlvwz0_internal : - Intrinsic<[llvm_x86amx_ty, llvm_x86amx_ty], - [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_ptr_ty, llvm_i64_ty], - []>; - def int_x86_t2rpntlvwz0t1_internal : - Intrinsic<[llvm_x86amx_ty, llvm_x86amx_ty], - [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_ptr_ty, llvm_i64_ty], - []>; - def int_x86_t2rpntlvwz1_internal : - Intrinsic<[llvm_x86amx_ty, llvm_x86amx_ty], - [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_ptr_ty, llvm_i64_ty], - []>; - def int_x86_t2rpntlvwz1t1_internal : - Intrinsic<[llvm_x86amx_ty, llvm_x86amx_ty], - [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_ptr_ty, llvm_i64_ty], - []>; - def int_x86_ttransposed_internal : - ClangBuiltin<"__builtin_ia32_ttransposed_internal">, - Intrinsic<[llvm_x86amx_ty], - [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty], []>; - def int_x86_ttdpbf16ps_internal : - ClangBuiltin<"__builtin_ia32_ttdpbf16ps_internal">, - Intrinsic<[llvm_x86amx_ty], - [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, - llvm_x86amx_ty, llvm_x86amx_ty, - llvm_x86amx_ty], []>; - def int_x86_ttdpfp16ps_internal : - ClangBuiltin<"__builtin_ia32_ttdpfp16ps_internal">, - Intrinsic<[llvm_x86amx_ty], - [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, - llvm_x86amx_ty, llvm_x86amx_ty, - llvm_x86amx_ty], []>; - def int_x86_ttcmmimfp16ps_internal : - ClangBuiltin<"__builtin_ia32_ttcmmimfp16ps_internal">, - Intrinsic<[llvm_x86amx_ty], - [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, - llvm_x86amx_ty, llvm_x86amx_ty, - llvm_x86amx_ty], []>; - def int_x86_ttcmmrlfp16ps_internal : - ClangBuiltin<"__builtin_ia32_ttcmmrlfp16ps_internal">, - Intrinsic<[llvm_x86amx_ty], - [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, - llvm_x86amx_ty, llvm_x86amx_ty, - llvm_x86amx_ty], []>; - def int_x86_tconjtcmmimfp16ps_internal : - ClangBuiltin<"__builtin_ia32_tconjtcmmimfp16ps_internal">, - Intrinsic<[llvm_x86amx_ty], - [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, - llvm_x86amx_ty, llvm_x86amx_ty, - llvm_x86amx_ty], []>; - def int_x86_tconjtfp16_internal : - ClangBuiltin<"__builtin_ia32_tconjtfp16_internal">, - Intrinsic<[llvm_x86amx_ty], - [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty], []>; - def int_x86_tcvtrowd2ps_internal : ClangBuiltin<"__builtin_ia32_tcvtrowd2ps_internal">, Intrinsic<[llvm_v16f32_ty], @@ -5775,20 +5680,11 @@ let TargetPrefix = "x86" in { Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; - def int_x86_ttmmultf32ps : ClangBuiltin<"__builtin_ia32_ttmmultf32ps">, - Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], - [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, - ImmArg<ArgIndex<2>>]>; def int_x86_tmmultf32ps_internal : ClangBuiltin<"__builtin_ia32_tmmultf32ps_internal">, Intrinsic<[llvm_x86amx_ty], [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_x86amx_ty, llvm_x86amx_ty], []>; - def int_x86_ttmmultf32ps_internal : - ClangBuiltin<"__builtin_ia32_ttmmultf32ps_internal">, - Intrinsic<[llvm_x86amx_ty], - [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, - llvm_x86amx_ty, llvm_x86amx_ty], []>; def int_x86_tdpbf8ps_internal : ClangBuiltin<"__builtin_ia32_tdpbf8ps_internal">, diff --git a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h index 3381e17..ccb77e7 100644 --- a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h @@ -79,7 +79,7 @@ struct CustomMappingTraits< } Args.push_back(Arg); } - io.mapRequired(Key.str().c_str(), V[Args]); + io.mapRequired(Key, V[Args]); } static void output( IO &io, @@ -91,7 +91,7 @@ struct CustomMappingTraits< Key += ','; Key += llvm::utostr(Arg); } - io.mapRequired(Key.c_str(), P.second); + io.mapRequired(Key, P.second); } } }; @@ -122,11 +122,11 @@ struct CustomMappingTraits<std::map<uint64_t, WholeProgramDevirtResolution>> { io.setError("key not an integer"); return; } - io.mapRequired(Key.str().c_str(), V[KeyInt]); + io.mapRequired(Key, V[KeyInt]); } static void output(IO &io, std::map<uint64_t, WholeProgramDevirtResolution> &V) { for (auto &P : V) - io.mapRequired(llvm::utostr(P.first).c_str(), P.second); + io.mapRequired(llvm::utostr(P.first), P.second); } }; @@ -215,7 +215,7 @@ namespace yaml { template <> struct CustomMappingTraits<GlobalValueSummaryMapTy> { static void inputOne(IO &io, StringRef Key, GlobalValueSummaryMapTy &V) { std::vector<GlobalValueSummaryYaml> GVSums; - io.mapRequired(Key.str().c_str(), GVSums); + io.mapRequired(Key, GVSums); uint64_t KeyInt; if (Key.getAsInteger(0, KeyInt)) { io.setError("key not an integer"); @@ -290,7 +290,7 @@ template <> struct CustomMappingTraits<GlobalValueSummaryMapTy> { } } if (!GVSums.empty()) - io.mapRequired(llvm::utostr(P.first).c_str(), GVSums); + io.mapRequired(llvm::utostr(P.first), GVSums); } } static void fixAliaseeLinks(GlobalValueSummaryMapTy &V) { @@ -313,12 +313,12 @@ template <> struct CustomMappingTraits<GlobalValueSummaryMapTy> { template <> struct CustomMappingTraits<TypeIdSummaryMapTy> { static void inputOne(IO &io, StringRef Key, TypeIdSummaryMapTy &V) { TypeIdSummary TId; - io.mapRequired(Key.str().c_str(), TId); + io.mapRequired(Key, TId); V.insert({GlobalValue::getGUIDAssumingExternalLinkage(Key), {Key, TId}}); } static void output(IO &io, TypeIdSummaryMapTy &V) { for (auto &TidIter : V) - io.mapRequired(TidIter.second.first.str().c_str(), TidIter.second.second); + io.mapRequired(TidIter.second.first, TidIter.second.second); } }; diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.td b/llvm/include/llvm/IR/RuntimeLibcalls.td index 7be1b65..24c1b03 100644 --- a/llvm/include/llvm/IR/RuntimeLibcalls.td +++ b/llvm/include/llvm/IR/RuntimeLibcalls.td @@ -1585,7 +1585,7 @@ def __aeabi_f2ulz : RuntimeLibcallImpl<FPTOUINT_F32_I64>; // CallingConv::ARM_AA // RTABI chapter 4.1.2, Table 7 def __aeabi_d2f : RuntimeLibcallImpl<FPROUND_F64_F32>; // CallingConv::ARM_AAPCS def __aeabi_d2h : RuntimeLibcallImpl<FPROUND_F64_F16>; // CallingConv::ARM_AAPCS -def __aeabi_f2d : RuntimeLibcallImpl<FPEXT_F32_F64>; // CallingConv::ARM_AAPCS +def __aeabi_f2d : RuntimeLibcallImpl<FPEXT_F32_F64>; // CallingConv::ARM_AAPCS // Integer to floating-point conversions. // RTABI chapter 4.1.2, Table 8 diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 581b4ad..c8196d8 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -90,7 +90,6 @@ LLVM_ABI void initializeDSELegacyPassPass(PassRegistry &); LLVM_ABI void initializeDXILMetadataAnalysisWrapperPassPass(PassRegistry &); LLVM_ABI void initializeDXILMetadataAnalysisWrapperPrinterPass(PassRegistry &); LLVM_ABI void initializeDXILResourceBindingWrapperPassPass(PassRegistry &); -LLVM_ABI void initializeDXILResourceImplicitBindingLegacyPass(PassRegistry &); LLVM_ABI void initializeDXILResourceTypeWrapperPassPass(PassRegistry &); LLVM_ABI void initializeDXILResourceWrapperPassPass(PassRegistry &); LLVM_ABI void initializeDeadMachineInstructionElimPass(PassRegistry &); diff --git a/llvm/include/llvm/MC/MCParser/MCAsmParser.h b/llvm/include/llvm/MC/MCParser/MCAsmParser.h index e3f44a0..5d74b76 100644 --- a/llvm/include/llvm/MC/MCParser/MCAsmParser.h +++ b/llvm/include/llvm/MC/MCParser/MCAsmParser.h @@ -209,28 +209,25 @@ public: MCInstPrinter *IP, MCAsmParserSemaCallback &SI) = 0; /// Emit a note at the location \p L, with the message \p Msg. - virtual void Note(SMLoc L, const Twine &Msg, - SMRange Range = std::nullopt) = 0; + virtual void Note(SMLoc L, const Twine &Msg, SMRange Range = {}) = 0; /// Emit a warning at the location \p L, with the message \p Msg. /// /// \return The return value is true, if warnings are fatal. - virtual bool Warning(SMLoc L, const Twine &Msg, - SMRange Range = std::nullopt) = 0; + virtual bool Warning(SMLoc L, const Twine &Msg, SMRange Range = {}) = 0; /// Return an error at the location \p L, with the message \p Msg. This /// may be modified before being emitted. /// /// \return The return value is always true, as an idiomatic convenience to /// clients. - bool Error(SMLoc L, const Twine &Msg, SMRange Range = std::nullopt); + bool Error(SMLoc L, const Twine &Msg, SMRange Range = {}); /// Emit an error at the location \p L, with the message \p Msg. /// /// \return The return value is always true, as an idiomatic convenience to /// clients. - virtual bool printError(SMLoc L, const Twine &Msg, - SMRange Range = std::nullopt) = 0; + virtual bool printError(SMLoc L, const Twine &Msg, SMRange Range = {}) = 0; bool hasPendingError() { return !PendingErrors.empty(); } @@ -255,7 +252,7 @@ public: const AsmToken &getTok() const; /// Report an error at the current lexer location. - bool TokError(const Twine &Msg, SMRange Range = std::nullopt); + bool TokError(const Twine &Msg, SMRange Range = {}); bool parseTokenLoc(SMLoc &Loc); bool parseToken(AsmToken::TokenKind T, const Twine &Msg = "unexpected token"); diff --git a/llvm/include/llvm/Object/ELF.h b/llvm/include/llvm/Object/ELF.h index 59f63eb..03d5ee2 100644 --- a/llvm/include/llvm/Object/ELF.h +++ b/llvm/include/llvm/Object/ELF.h @@ -278,9 +278,46 @@ private: std::vector<Elf_Shdr> FakeSections; SmallString<0> FakeSectionStrings; + // When the number of program headers is >= PN_XNUM, the actual number is + // contained in the sh_info field of the section header at index 0. + std::optional<uint32_t> RealPhNum; + // When the number of section headers is >= SHN_LORESERVE, the actual number + // is contained in the sh_size field of the section header at index 0. + std::optional<uint64_t> RealShNum; + // When the section index of the section name table is >= SHN_LORESERVE, the + // actual number is contained in the sh_link field of the section header at + // index 0. + std::optional<uint32_t> RealShStrNdx; + ELFFile(StringRef Object); + Error readShdrZero(); + public: + Expected<uint32_t> getPhNum() const { + if (!RealPhNum) { + if (Error E = const_cast<ELFFile<ELFT> *>(this)->readShdrZero()) + return std::move(E); + } + return *RealPhNum; + } + + Expected<uint64_t> getShNum() const { + if (!RealShNum) { + if (Error E = const_cast<ELFFile<ELFT> *>(this)->readShdrZero()) + return std::move(E); + } + return *RealShNum; + } + + Expected<uint32_t> getShStrNdx() const { + if (!RealShStrNdx) { + if (Error E = const_cast<ELFFile<ELFT> *>(this)->readShdrZero()) + return std::move(E); + } + return *RealShStrNdx; + } + const Elf_Ehdr &getHeader() const { return *reinterpret_cast<const Elf_Ehdr *>(base()); } @@ -379,22 +416,26 @@ public: /// Iterate over program header table. Expected<Elf_Phdr_Range> program_headers() const { - if (getHeader().e_phnum && getHeader().e_phentsize != sizeof(Elf_Phdr)) + uint32_t NumPh; + if (Expected<uint32_t> PhNumOrErr = getPhNum()) + NumPh = *PhNumOrErr; + else + return PhNumOrErr.takeError(); + if (NumPh && getHeader().e_phentsize != sizeof(Elf_Phdr)) return createError("invalid e_phentsize: " + Twine(getHeader().e_phentsize)); - uint64_t HeadersSize = - (uint64_t)getHeader().e_phnum * getHeader().e_phentsize; + uint64_t HeadersSize = (uint64_t)NumPh * getHeader().e_phentsize; uint64_t PhOff = getHeader().e_phoff; if (PhOff + HeadersSize < PhOff || PhOff + HeadersSize > getBufSize()) return createError("program headers are longer than binary of size " + Twine(getBufSize()) + ": e_phoff = 0x" + Twine::utohexstr(getHeader().e_phoff) + - ", e_phnum = " + Twine(getHeader().e_phnum) + + ", e_phnum = " + Twine(NumPh) + ", e_phentsize = " + Twine(getHeader().e_phentsize)); auto *Begin = reinterpret_cast<const Elf_Phdr *>(base() + PhOff); - return ArrayRef(Begin, Begin + getHeader().e_phnum); + return ArrayRef(Begin, Begin + NumPh); } /// Get an iterator over notes in a program header. @@ -772,19 +813,15 @@ template <class ELFT> Expected<StringRef> ELFFile<ELFT>::getSectionStringTable(Elf_Shdr_Range Sections, WarningHandler WarnHandler) const { - uint32_t Index = getHeader().e_shstrndx; - if (Index == ELF::SHN_XINDEX) { - // If the section name string table section index is greater than - // or equal to SHN_LORESERVE, then the actual index of the section name - // string table section is contained in the sh_link field of the section - // header at index 0. - if (Sections.empty()) - return createError( - "e_shstrndx == SHN_XINDEX, but the section header table is empty"); + Expected<uint32_t> ShStrNdxOrErr = getShStrNdx(); + if (!ShStrNdxOrErr) + return ShStrNdxOrErr.takeError(); - Index = Sections[0].sh_link; - } + if (*ShStrNdxOrErr == ELF::SHN_XINDEX && Sections.empty()) + return createError( + "e_shstrndx == SHN_XINDEX, but the section header table is empty"); + uint32_t Index = *ShStrNdxOrErr; // There is no section name string table. Return FakeSectionStrings which // is non-empty if we have created fake sections. if (!Index) @@ -891,6 +928,35 @@ Expected<uint64_t> ELFFile<ELFT>::getDynSymtabSize() const { template <class ELFT> ELFFile<ELFT>::ELFFile(StringRef Object) : Buf(Object) {} +template <class ELFT> Error ELFFile<ELFT>::readShdrZero() { + const Elf_Ehdr &Header = getHeader(); + + if ((Header.e_phnum == ELF::PN_XNUM || Header.e_shnum == 0 || + Header.e_shstrndx == ELF::SHN_XINDEX) && + Header.e_shoff != 0) { + // Pretend we have section 0 or sections() would call getShNum and thus + // become an infinite recursion. + RealShNum = 1; + auto SecOrErr = getSection(0); + if (!SecOrErr) { + RealShNum = std::nullopt; + return SecOrErr.takeError(); + } + + RealPhNum = + Header.e_phnum == ELF::PN_XNUM ? (*SecOrErr)->sh_info : Header.e_phnum; + RealShNum = Header.e_shnum == 0 ? (*SecOrErr)->sh_size : Header.e_shnum; + RealShStrNdx = Header.e_shstrndx == ELF::SHN_XINDEX ? (*SecOrErr)->sh_link + : Header.e_shstrndx; + } else { + RealPhNum = Header.e_phnum; + RealShNum = Header.e_shnum; + RealShStrNdx = Header.e_shstrndx; + } + + return Error::success(); +} + template <class ELFT> Expected<ELFFile<ELFT>> ELFFile<ELFT>::create(StringRef Object) { if (sizeof(Elf_Ehdr) > Object.size()) @@ -956,9 +1022,11 @@ Expected<typename ELFT::ShdrRange> ELFFile<ELFT>::sections() const { const Elf_Shdr *First = reinterpret_cast<const Elf_Shdr *>(base() + SectionTableOffset); - uintX_t NumSections = getHeader().e_shnum; - if (NumSections == 0) - NumSections = First->sh_size; + uintX_t NumSections = 0; + if (Expected<uint64_t> ShNumOrErr = getShNum()) + NumSections = *ShNumOrErr; + else + return ShNumOrErr.takeError(); if (NumSections > UINT64_MAX / sizeof(Elf_Shdr)) return createError("invalid number of sections specified in the NULL " diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h index e9a417d..467ab6f 100644 --- a/llvm/include/llvm/Object/ELFTypes.h +++ b/llvm/include/llvm/Object/ELFTypes.h @@ -834,30 +834,32 @@ struct BBAddrMap { bool OmitBBEntries : 1; bool CallsiteEndOffsets : 1; bool BBHash : 1; + bool PostLinkCfg : 1; bool hasPGOAnalysis() const { return FuncEntryCount || BBFreq || BrProb; } bool hasPGOAnalysisBBData() const { return BBFreq || BrProb; } // Encodes to minimum bit width representation. - uint8_t encode() const { - return (static_cast<uint8_t>(FuncEntryCount) << 0) | - (static_cast<uint8_t>(BBFreq) << 1) | - (static_cast<uint8_t>(BrProb) << 2) | - (static_cast<uint8_t>(MultiBBRange) << 3) | - (static_cast<uint8_t>(OmitBBEntries) << 4) | - (static_cast<uint8_t>(CallsiteEndOffsets) << 5) | - (static_cast<uint8_t>(BBHash) << 6); + uint16_t encode() const { + return (static_cast<uint16_t>(FuncEntryCount) << 0) | + (static_cast<uint16_t>(BBFreq) << 1) | + (static_cast<uint16_t>(BrProb) << 2) | + (static_cast<uint16_t>(MultiBBRange) << 3) | + (static_cast<uint16_t>(OmitBBEntries) << 4) | + (static_cast<uint16_t>(CallsiteEndOffsets) << 5) | + (static_cast<uint16_t>(BBHash) << 6) | + (static_cast<uint16_t>(PostLinkCfg) << 7); } // Decodes from minimum bit width representation and validates no // unnecessary bits are used. - static Expected<Features> decode(uint8_t Val) { + static Expected<Features> decode(uint16_t Val) { Features Feat{ static_cast<bool>(Val & (1 << 0)), static_cast<bool>(Val & (1 << 1)), static_cast<bool>(Val & (1 << 2)), static_cast<bool>(Val & (1 << 3)), static_cast<bool>(Val & (1 << 4)), static_cast<bool>(Val & (1 << 5)), - static_cast<bool>(Val & (1 << 6))}; + static_cast<bool>(Val & (1 << 6)), static_cast<bool>(Val & (1 << 7))}; if (Feat.encode() != Val) return createStringError( std::error_code(), "invalid encoding for BBAddrMap::Features: 0x%x", @@ -867,10 +869,11 @@ struct BBAddrMap { bool operator==(const Features &Other) const { return std::tie(FuncEntryCount, BBFreq, BrProb, MultiBBRange, - OmitBBEntries, CallsiteEndOffsets, BBHash) == + OmitBBEntries, CallsiteEndOffsets, BBHash, PostLinkCfg) == std::tie(Other.FuncEntryCount, Other.BBFreq, Other.BrProb, Other.MultiBBRange, Other.OmitBBEntries, - Other.CallsiteEndOffsets, Other.BBHash); + Other.CallsiteEndOffsets, Other.BBHash, + Other.PostLinkCfg); } }; @@ -1010,23 +1013,30 @@ struct PGOAnalysisMap { /// probability associated with it. struct SuccessorEntry { /// Unique ID of this successor basic block. - uint32_t ID; + uint32_t ID = 0; /// Branch Probability of the edge to this successor taken from MBPI. BranchProbability Prob; + /// Raw edge count from the post link profile (e.g., from bolt or + /// propeller). + uint64_t PostLinkFreq = 0; bool operator==(const SuccessorEntry &Other) const { - return std::tie(ID, Prob) == std::tie(Other.ID, Other.Prob); + return std::tie(ID, Prob, PostLinkFreq) == + std::tie(Other.ID, Other.Prob, Other.PostLinkFreq); } }; /// Block frequency taken from MBFI BlockFrequency BlockFreq; + /// Raw block count taken from the post link profile (e.g., from bolt or + /// propeller). + uint64_t PostLinkBlockFreq = 0; /// List of successors of the current block llvm::SmallVector<SuccessorEntry, 2> Successors; bool operator==(const PGOBBEntry &Other) const { - return std::tie(BlockFreq, Successors) == - std::tie(Other.BlockFreq, Other.Successors); + return std::tie(BlockFreq, PostLinkBlockFreq, Successors) == + std::tie(Other.BlockFreq, PostLinkBlockFreq, Other.Successors); } }; diff --git a/llvm/include/llvm/Object/SFrameParser.h b/llvm/include/llvm/Object/SFrameParser.h index 3ce5d70..23298357 100644 --- a/llvm/include/llvm/Object/SFrameParser.h +++ b/llvm/include/llvm/Object/SFrameParser.h @@ -90,7 +90,7 @@ public: uint32_t Idx, uint32_t Size, uint64_t Offset) : Data(Data), FREType(FREType), Idx(Idx), Size(Size), Offset(Offset) {} - Error inc(); + LLVM_ABI Error inc(); const FrameRowEntry &operator*() const { return FRE; } friend bool operator==(const FallibleFREIterator &LHS, diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h index a7c7c7c..a8236ca 100644 --- a/llvm/include/llvm/ObjectYAML/ELFYAML.h +++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h @@ -166,7 +166,7 @@ struct BBAddrMapEntry { std::optional<llvm::yaml::Hex64> Hash; }; uint8_t Version; - llvm::yaml::Hex8 Feature; + llvm::yaml::Hex16 Feature; struct BBRangeEntry { llvm::yaml::Hex64 BaseAddress; @@ -203,8 +203,10 @@ struct PGOAnalysisMapEntry { struct SuccessorEntry { uint32_t ID; llvm::yaml::Hex32 BrProb; + std::optional<uint32_t> PostLinkBrFreq; }; std::optional<uint64_t> BBFreq; + std::optional<uint32_t> PostLinkBBFreq; std::optional<std::vector<SuccessorEntry>> Successors; }; std::optional<uint64_t> FuncEntryCount; diff --git a/llvm/include/llvm/ProfileData/MemProfYAML.h b/llvm/include/llvm/ProfileData/MemProfYAML.h index d66e16d..c55f780 100644 --- a/llvm/include/llvm/ProfileData/MemProfYAML.h +++ b/llvm/include/llvm/ProfileData/MemProfYAML.h @@ -141,7 +141,7 @@ template <> struct CustomMappingTraits<memprof::PortableMemInfoBlock> { #define MIBEntryDef(NameTag, Name, Type) \ if (KeyStr == #Name) { \ uint64_t Value; \ - Io.mapRequired(KeyStr.str().c_str(), Value); \ + Io.mapRequired(KeyStr, Value); \ MIB.Name = static_cast<Type>(Value); \ MIB.Schema.set(llvm::to_underlying(memprof::Meta::Name)); \ return; \ diff --git a/llvm/include/llvm/Support/BranchProbability.h b/llvm/include/llvm/Support/BranchProbability.h index 42fe225..b15d6e1 100644 --- a/llvm/include/llvm/Support/BranchProbability.h +++ b/llvm/include/llvm/Support/BranchProbability.h @@ -97,6 +97,9 @@ public: /// \return \c Num divided by \c this. LLVM_ABI uint64_t scaleByInverse(uint64_t Num) const; + /// Compute pow(Probability, N). + BranchProbability pow(unsigned N) const; + BranchProbability &operator+=(BranchProbability RHS) { assert(N != UnknownN && RHS.N != UnknownN && "Unknown probability cannot participate in arithmetics."); diff --git a/llvm/include/llvm/Support/ELFAttributeParser.h b/llvm/include/llvm/Support/ELFAttributeParser.h index 97350ed..c2ad812 100644 --- a/llvm/include/llvm/Support/ELFAttributeParser.h +++ b/llvm/include/llvm/Support/ELFAttributeParser.h @@ -17,7 +17,7 @@ namespace llvm { class ELFAttributeParser { public: - virtual ~ELFAttributeParser() {} + virtual ~ELFAttributeParser() = default; virtual Error parse(ArrayRef<uint8_t> Section, llvm::endianness Endian) { return llvm::Error::success(); diff --git a/llvm/include/llvm/Support/GenericLoopInfo.h b/llvm/include/llvm/Support/GenericLoopInfo.h index 2775a87..b6bb360 100644 --- a/llvm/include/llvm/Support/GenericLoopInfo.h +++ b/llvm/include/llvm/Support/GenericLoopInfo.h @@ -615,6 +615,17 @@ public: return L ? L->getLoopDepth() : 0; } + /// \brief Find the innermost loop containing both given loops. + /// + /// \returns the innermost loop containing both \p A and \p B + /// or nullptr if there is no such loop. + LoopT *getSmallestCommonLoop(LoopT *A, LoopT *B) const; + /// \brief Find the innermost loop containing both given blocks. + /// + /// \returns the innermost loop containing both \p A and \p B + /// or nullptr if there is no such loop. + LoopT *getSmallestCommonLoop(BlockT *A, BlockT *B) const; + // True if the block is a loop header node bool isLoopHeader(const BlockT *BB) const { const LoopT *L = getLoopFor(BB); diff --git a/llvm/include/llvm/Support/GenericLoopInfoImpl.h b/llvm/include/llvm/Support/GenericLoopInfoImpl.h index 6fc508b..5416780 100644 --- a/llvm/include/llvm/Support/GenericLoopInfoImpl.h +++ b/llvm/include/llvm/Support/GenericLoopInfoImpl.h @@ -355,7 +355,7 @@ void LoopBase<BlockT, LoopT>::verifyLoop() const { if (BB == getHeader()) { assert(!OutsideLoopPreds.empty() && "Loop is unreachable!"); } else if (!OutsideLoopPreds.empty()) { - // A non-header loop shouldn't be reachable from outside the loop, + // A non-header loop block shouldn't be reachable from outside the loop, // though it is permitted if the predecessor is not itself actually // reachable. BlockT *EntryBB = &BB->getParent()->front(); @@ -645,6 +645,36 @@ LoopInfoBase<BlockT, LoopT>::getLoopsInReverseSiblingPreorder() const { return PreOrderLoops; } +template <class BlockT, class LoopT> +LoopT *LoopInfoBase<BlockT, LoopT>::getSmallestCommonLoop(LoopT *A, + LoopT *B) const { + if (!A || !B) + return nullptr; + + // If lops A and B have different depth replace them with parent loop + // until they have the same depth. + while (A->getLoopDepth() > B->getLoopDepth()) + A = A->getParentLoop(); + while (B->getLoopDepth() > A->getLoopDepth()) + B = B->getParentLoop(); + + // Loops A and B are at same depth but may be disjoint, replace them with + // parent loops until we find loop that contains both or we run out of + // parent loops. + while (A != B) { + A = A->getParentLoop(); + B = B->getParentLoop(); + } + + return A; +} + +template <class BlockT, class LoopT> +LoopT *LoopInfoBase<BlockT, LoopT>::getSmallestCommonLoop(BlockT *A, + BlockT *B) const { + return getSmallestCommonLoop(getLoopFor(A), getLoopFor(B)); +} + // Debugging template <class BlockT, class LoopT> void LoopInfoBase<BlockT, LoopT>::print(raw_ostream &OS) const { diff --git a/llvm/include/llvm/Support/GraphWriter.h b/llvm/include/llvm/Support/GraphWriter.h index 3bef75c..43d9b0c 100644 --- a/llvm/include/llvm/Support/GraphWriter.h +++ b/llvm/include/llvm/Support/GraphWriter.h @@ -128,7 +128,7 @@ public: DTraits = DOTTraits(SN); RenderUsingHTML = DTraits.renderNodesUsingHTML(); } - virtual ~GraphWriterBase() {} + virtual ~GraphWriterBase() = default; void writeGraph(const std::string &Title = "") { // Output the header for the graph... @@ -369,7 +369,7 @@ class GraphWriter : public GraphWriterBase<GraphType, GraphWriter<GraphType>> { public: GraphWriter(raw_ostream &o, const GraphType &g, bool SN) : GraphWriterBase<GraphType, GraphWriter<GraphType>>(o, g, SN) {} - ~GraphWriter() override {} + ~GraphWriter() override = default; }; template <typename GraphType> diff --git a/llvm/include/llvm/Support/JSON.h b/llvm/include/llvm/Support/JSON.h index d8c6de4..a973c56 100644 --- a/llvm/include/llvm/Support/JSON.h +++ b/llvm/include/llvm/Support/JSON.h @@ -154,7 +154,7 @@ public: LLVM_ABI const json::Array *getArray(StringRef K) const; LLVM_ABI json::Array *getArray(StringRef K); - friend bool operator==(const Object &LHS, const Object &RHS); + friend LLVM_ABI bool operator==(const Object &LHS, const Object &RHS); }; LLVM_ABI bool operator==(const Object &LHS, const Object &RHS); inline bool operator!=(const Object &LHS, const Object &RHS) { diff --git a/llvm/include/llvm/Support/LEB128.h b/llvm/include/llvm/Support/LEB128.h index 898b4ea..4e2262fb 100644 --- a/llvm/include/llvm/Support/LEB128.h +++ b/llvm/include/llvm/Support/LEB128.h @@ -29,8 +29,7 @@ inline unsigned encodeSLEB128(int64_t Value, raw_ostream &OS, uint8_t Byte = Value & 0x7f; // NOTE: this assumes that this signed shift is an arithmetic right shift. Value >>= 7; - More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) || - ((Value == -1) && ((Byte & 0x40) != 0)))); + More = Value != ((Byte & 0x40) ? -1 : 0); Count++; if (More || Count < PadTo) Byte |= 0x80; // Mark this byte to show that more bytes will follow. @@ -58,8 +57,7 @@ inline unsigned encodeSLEB128(int64_t Value, uint8_t *p, unsigned PadTo = 0) { uint8_t Byte = Value & 0x7f; // NOTE: this assumes that this signed shift is an arithmetic right shift. Value >>= 7; - More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) || - ((Value == -1) && ((Byte & 0x40) != 0)))); + More = Value != ((Byte & 0x40) ? -1 : 0); Count++; if (More || Count < PadTo) Byte |= 0x80; // Mark this byte to show that more bytes will follow. diff --git a/llvm/include/llvm/Support/SMLoc.h b/llvm/include/llvm/Support/SMLoc.h index c80969b..b7ae6e4 100644 --- a/llvm/include/llvm/Support/SMLoc.h +++ b/llvm/include/llvm/Support/SMLoc.h @@ -15,7 +15,6 @@ #define LLVM_SUPPORT_SMLOC_H #include <cassert> -#include <optional> namespace llvm { @@ -50,7 +49,6 @@ public: SMLoc Start, End; SMRange() = default; - SMRange(std::nullopt_t) {} SMRange(SMLoc St, SMLoc En) : Start(St), End(En) { assert(Start.isValid() == End.isValid() && "Start and End should either both be valid or both be invalid!"); diff --git a/llvm/include/llvm/Support/SourceMgr.h b/llvm/include/llvm/Support/SourceMgr.h index 8320006..43f7e27 100644 --- a/llvm/include/llvm/Support/SourceMgr.h +++ b/llvm/include/llvm/Support/SourceMgr.h @@ -103,7 +103,7 @@ private: public: /// Create new source manager without support for include files. - SourceMgr(); + LLVM_ABI SourceMgr(); /// Create new source manager with the capability of finding include files /// via the provided file system. explicit SourceMgr(IntrusiveRefCntPtr<vfs::FileSystem> FS); @@ -111,10 +111,10 @@ public: SourceMgr &operator=(const SourceMgr &) = delete; SourceMgr(SourceMgr &&); SourceMgr &operator=(SourceMgr &&); - ~SourceMgr(); + LLVM_ABI ~SourceMgr(); IntrusiveRefCntPtr<vfs::FileSystem> getVirtualFileSystem() const; - void setVirtualFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS); + LLVM_ABI void setVirtualFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS); /// Return the include directories of this source manager. ArrayRef<std::string> getIncludeDirs() const { return IncludeDirectories; } diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h index c8911a0..dbd5a5c 100644 --- a/llvm/include/llvm/Support/VirtualFileSystem.h +++ b/llvm/include/llvm/Support/VirtualFileSystem.h @@ -1116,8 +1116,9 @@ protected: /// Collect all pairs of <virtual path, real path> entries from the /// \p VFS. This is used by the module dependency collector to forward /// the entries into the reproducer output VFS YAML file. -void collectVFSEntries(RedirectingFileSystem &VFS, - SmallVectorImpl<YAMLVFSEntry> &CollectedEntries); +LLVM_ABI void +collectVFSEntries(RedirectingFileSystem &VFS, + SmallVectorImpl<YAMLVFSEntry> &CollectedEntries); class YAMLVFSWriter { std::vector<YAMLVFSEntry> Mappings; diff --git a/llvm/include/llvm/Support/VirtualOutputBackend.h b/llvm/include/llvm/Support/VirtualOutputBackend.h index 85caa021..78ed4b9b 100644 --- a/llvm/include/llvm/Support/VirtualOutputBackend.h +++ b/llvm/include/llvm/Support/VirtualOutputBackend.h @@ -32,7 +32,7 @@ namespace llvm::vfs { /// If virtual functions are added here, also add them to \a /// ProxyOutputBackend. class OutputBackend : public RefCountedBase<OutputBackend> { - virtual void anchor(); + LLVM_ABI virtual void anchor(); public: /// Get a backend that points to the same destination as this one but that @@ -47,7 +47,7 @@ public: /// have been customized). /// /// Thread-safe. - Expected<OutputFile> + LLVM_ABI Expected<OutputFile> createFile(const Twine &Path, std::optional<OutputConfig> Config = std::nullopt); diff --git a/llvm/include/llvm/Support/VirtualOutputBackends.h b/llvm/include/llvm/Support/VirtualOutputBackends.h index 219bc30..13a9611 100644 --- a/llvm/include/llvm/Support/VirtualOutputBackends.h +++ b/llvm/include/llvm/Support/VirtualOutputBackends.h @@ -77,14 +77,14 @@ private: /// An output backend that creates files on disk, wrapping APIs in sys::fs. class OnDiskOutputBackend : public OutputBackend { - void anchor() override; + LLVM_ABI void anchor() override; protected: IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override { return clone(); } - Expected<std::unique_ptr<OutputFileImpl>> + LLVM_ABI Expected<std::unique_ptr<OutputFileImpl>> createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override; public: diff --git a/llvm/include/llvm/Support/VirtualOutputError.h b/llvm/include/llvm/Support/VirtualOutputError.h index 2293ff9..44590a1 100644 --- a/llvm/include/llvm/Support/VirtualOutputError.h +++ b/llvm/include/llvm/Support/VirtualOutputError.h @@ -43,7 +43,7 @@ public: void log(raw_ostream &OS) const override; // Used by ErrorInfo::classID. - static char ID; + LLVM_ABI static char ID; OutputError(const Twine &OutputPath, std::error_code EC) : ErrorInfo<OutputError, ECError>(EC), OutputPath(OutputPath.str()) { @@ -99,7 +99,7 @@ public: void log(raw_ostream &OS) const override; // Used by ErrorInfo::classID. - static char ID; + LLVM_ABI static char ID; TempFileOutputError(const Twine &TempPath, const Twine &OutputPath, std::error_code EC) diff --git a/llvm/include/llvm/Support/VirtualOutputFile.h b/llvm/include/llvm/Support/VirtualOutputFile.h index dd50437..d53701c 100644 --- a/llvm/include/llvm/Support/VirtualOutputFile.h +++ b/llvm/include/llvm/Support/VirtualOutputFile.h @@ -80,13 +80,13 @@ public: /// /// If there's an open proxy from \a createProxy(), calls \a discard() to /// clean up temporaries followed by \a report_fatal_error(). - Error keep(); + LLVM_ABI Error keep(); /// Discard an output, cleaning up any temporary state. Errors if clean-up /// fails. /// /// If it has already been closed, calls \a report_fatal_error(). - Error discard(); + LLVM_ABI Error discard(); /// Discard the output when destroying it if it's still open, sending the /// result to \a Handler. @@ -98,7 +98,7 @@ public: /// producer. Errors if there's already a proxy. The proxy must be deleted /// before calling \a keep(). The proxy will crash if it's written to after /// calling \a discard(). - Expected<std::unique_ptr<raw_pwrite_stream>> createProxy(); + LLVM_ABI Expected<std::unique_ptr<raw_pwrite_stream>> createProxy(); bool hasOpenProxy() const { return OpenProxy; } @@ -132,7 +132,7 @@ public: private: /// Destroy \a Impl. Reports fatal error if the file is open and there's no /// handler from \a discardOnDestroy(). - void destroy(); + LLVM_ABI void destroy(); OutputFile &moveFrom(OutputFile &O) { Path = std::move(O.Path); Impl = std::move(O.Impl); diff --git a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h index 4aa6c01..6f6f65d 100644 --- a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h +++ b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h @@ -511,7 +511,6 @@ enum OperandEncoding { ENCODINGS ENCODING_max }; ENUM_ENTRY(TYPE_VK, "mask register") \ ENUM_ENTRY(TYPE_VK_PAIR, "mask register pair") \ ENUM_ENTRY(TYPE_TMM, "tile") \ - ENUM_ENTRY(TYPE_TMM_PAIR, "tile pair") \ ENUM_ENTRY(TYPE_SEGMENTREG, "Segment register operand") \ ENUM_ENTRY(TYPE_DEBUGREG, "Debug register operand") \ ENUM_ENTRY(TYPE_CONTROLREG, "Control register operand") \ diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h index 3d36f41..b53b28d 100644 --- a/llvm/include/llvm/Support/YAMLTraits.h +++ b/llvm/include/llvm/Support/YAMLTraits.h @@ -1921,12 +1921,12 @@ template <typename T> struct StdMapStringCustomMappingTraitsImpl { using map_type = std::map<std::string, T>; static void inputOne(IO &io, StringRef key, map_type &v) { - io.mapRequired(key.str().c_str(), v[std::string(key)]); + io.mapRequired(key, v[std::string(key)]); } static void output(IO &io, map_type &v) { for (auto &p : v) - io.mapRequired(p.first.c_str(), p.second); + io.mapRequired(p.first, p.second); } }; diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def index a94eab1..78cf464 100644 --- a/llvm/include/llvm/TargetParser/X86TargetParser.def +++ b/llvm/include/llvm/TargetParser/X86TargetParser.def @@ -268,7 +268,6 @@ X86_FEATURE_COMPAT(AVX10_2_512, "avx10.2-512", 0) X86_FEATURE (MOVRS, "movrs") X86_FEATURE (ZU, "zu") X86_FEATURE (AMX_FP8, "amx-fp8") -X86_FEATURE (AMX_TRANSPOSE, "amx-transpose") X86_FEATURE (AMX_MOVRS, "amx-movrs") X86_FEATURE (AMX_AVX512, "amx-avx512") X86_FEATURE (AMX_TF32, "amx-tf32") diff --git a/llvm/include/llvm/Transforms/IPO/InferFunctionAttrs.h b/llvm/include/llvm/Transforms/IPO/InferFunctionAttrs.h index 8addf49..272b960 100644 --- a/llvm/include/llvm/Transforms/IPO/InferFunctionAttrs.h +++ b/llvm/include/llvm/Transforms/IPO/InferFunctionAttrs.h @@ -23,7 +23,7 @@ class Module; /// A pass which infers function attributes from the names and signatures of /// function declarations in a module. struct InferFunctionAttrsPass : PassInfoMixin<InferFunctionAttrsPass> { - PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; } diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h index ced446d..9dcd4b5 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h +++ b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h @@ -26,8 +26,6 @@ namespace llvm { -LLVM_ABI extern cl::opt<bool> DebugInfoCorrelate; - class Function; class Instruction; class Module; diff --git a/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h b/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h index a8a09fb..346e7f0 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h +++ b/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h @@ -33,7 +33,7 @@ class FileSystem; /// appends globals to llvm.compiler.used. class SanitizerCoveragePass : public PassInfoMixin<SanitizerCoveragePass> { public: - explicit SanitizerCoveragePass( + LLVM_ABI explicit SanitizerCoveragePass( SanitizerCoverageOptions Options = SanitizerCoverageOptions(), IntrusiveRefCntPtr<vfs::FileSystem> VFS = nullptr, const std::vector<std::string> &AllowlistFiles = {}, diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h index e677cbf..49885b7 100644 --- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h +++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h @@ -19,6 +19,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SetVector.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Printable.h" @@ -262,6 +263,34 @@ LLVM_ABI BasicBlock *SplitEdge(BasicBlock *From, BasicBlock *To, MemorySSAUpdater *MSSAU = nullptr, const Twine &BBName = ""); +/// \brief Create a new intermediate target block for a callbr edge. +/// +/// Create a new basic block between a callbr instruction and one of its +/// successors. The new block replaces the original successor in the callbr +/// instruction and unconditionally branches to the original successor. This +/// is useful for normalizing control flow, e.g., when transforming +/// irreducible loops. +/// +/// \param CallBrBlock block containing the callbr instruction +/// \param Succ original successor block +/// \param SuccIdx index of the original successor in the callbr +/// instruction +/// \param DTU optional \p DomTreeUpdater for updating the +/// dominator tree +/// \param CI optional \p CycleInfo for updating cycle membership +/// \param LI optional \p LoopInfo for updating loop membership +/// \param UpdatedLI optional output flag indicating if \p LoopInfo has +/// been updated +/// +/// \returns newly created intermediate target block +/// +/// \note This function updates PHI nodes, dominator tree, loop info, and +/// cycle info as needed. +LLVM_ABI BasicBlock * +SplitCallBrEdge(BasicBlock *CallBrBlock, BasicBlock *Succ, unsigned SuccIdx, + DomTreeUpdater *DTU = nullptr, CycleInfo *CI = nullptr, + LoopInfo *LI = nullptr, bool *UpdatedLI = nullptr); + /// Sets the unwind edge of an instruction to a particular successor. LLVM_ABI void setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ); diff --git a/llvm/include/llvm/Transforms/Utils/ControlFlowUtils.h b/llvm/include/llvm/Transforms/Utils/ControlFlowUtils.h index 810fef2..17cde82 100644 --- a/llvm/include/llvm/Transforms/Utils/ControlFlowUtils.h +++ b/llvm/include/llvm/Transforms/Utils/ControlFlowUtils.h @@ -15,10 +15,13 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/IR/CycleInfo.h" namespace llvm { class BasicBlock; +class CallBrInst; +class LoopInfo; class DomTreeUpdater; /// Given a set of branch descriptors [BB, Succ0, Succ1], create a "hub" such @@ -104,7 +107,8 @@ struct ControlFlowHub { : BB(BB), Succ0(Succ0), Succ1(Succ1) {} }; - void addBranch(BasicBlock *BB, BasicBlock *Succ0, BasicBlock *Succ1) { + void addBranch(BasicBlock *BB, BasicBlock *Succ0, + BasicBlock *Succ1 = nullptr) { assert(BB); assert(Succ0 || Succ1); Branches.emplace_back(BB, Succ0, Succ1); diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h index 2d2355d..86eb213 100644 --- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -365,6 +365,40 @@ LLVM_ABI bool setLoopEstimatedTripCount( Loop *L, unsigned EstimatedTripCount, std::optional<unsigned> EstimatedLoopInvocationWeight = std::nullopt); +/// Based on branch weight metadata, return either: +/// - An unknown probability if the implementation is unable to handle the loop +/// form of \p L (e.g., \p L must have a latch block that controls the loop +/// exit). +/// - The probability \c P that, at the end of any iteration, the latch of \p L +/// will start another iteration such that `1 - P` is the probability of +/// exiting the loop. +BranchProbability getLoopProbability(Loop *L); + +/// Set branch weight metadata for the latch of \p L to indicate that, at the +/// end of any iteration, \p P and `1 - P` are the probabilities of starting +/// another iteration and exiting the loop, respectively. Return false if the +/// implementation is unable to handle the loop form of \p L (e.g., \p L must +/// have a latch block that controls the loop exit). Otherwise, return true. +bool setLoopProbability(Loop *L, BranchProbability P); + +/// Based on branch weight metadata, return either: +/// - An unknown probability if the implementation cannot extract the +/// probability (e.g., \p B must have exactly two target labels, so it must be +/// a conditional branch). +/// - The probability \c P that control flows from \p B to its first target +/// label such that `1 - P` is the probability of control flowing to its +/// second target label, or vice-versa if \p ForFirstTarget is false. +BranchProbability getBranchProbability(BranchInst *B, bool ForFirstTarget); + +/// Set branch weight metadata for \p B to indicate that \p P and `1 - P` are +/// the probabilities of control flowing to its first and second target labels, +/// respectively, or vice-versa if \p ForFirstTarget is false. Return false if +/// the implementation cannot set the probability (e.g., \p B must have exactly +/// two target labels, so it must be a conditional branch). Otherwise, return +/// true. +bool setBranchProbability(BranchInst *B, BranchProbability P, + bool ForFirstTarget); + /// Check inner loop (L) backedge count is known to be invariant on all /// iterations of its outer loop. If the loop has no parent, this is trivially /// true. diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h index 871c13d..a3efc43 100644 --- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h +++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h @@ -97,7 +97,9 @@ LLVM_ABI bool UnrollRuntimeLoopRemainder( LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC, const TargetTransformInfo *TTI, bool PreserveLCSSA, unsigned SCEVExpansionBudget, bool RuntimeUnrollMultiExit, - Loop **ResultLoop = nullptr); + Loop **ResultLoop = nullptr, + std::optional<unsigned> OriginalTripCount = std::nullopt, + BranchProbability OriginalLoopProb = BranchProbability::getUnknown()); LLVM_ABI LoopUnrollResult UnrollAndJamLoop( Loop *L, unsigned Count, unsigned TripCount, unsigned TripMultiple, |