diff options
Diffstat (limited to 'llvm/include')
30 files changed, 392 insertions, 144 deletions
diff --git a/llvm/include/llvm/ADT/BitVector.h b/llvm/include/llvm/ADT/BitVector.h index 83350e6..9e81a4b 100644 --- a/llvm/include/llvm/ADT/BitVector.h +++ b/llvm/include/llvm/ADT/BitVector.h @@ -570,10 +570,7 @@ public: template <class F, class... ArgTys> static BitVector &apply(F &&f, BitVector &Out, BitVector const &Arg, ArgTys const &...Args) { - assert(llvm::all_of( - std::initializer_list<unsigned>{Args.size()...}, - [&Arg](auto const &BV) { return Arg.size() == BV; }) && - "consistent sizes"); + assert(((Arg.size() == Args.size()) && ...) && "consistent sizes"); Out.resize(Arg.size()); for (size_type I = 0, E = Arg.Bits.size(); I != E; ++I) Out.Bits[I] = f(Arg.Bits[I], Args.Bits[I]...); diff --git a/llvm/include/llvm/ADT/BitmaskEnum.h b/llvm/include/llvm/ADT/BitmaskEnum.h index 7214f25..d464cbc 100644 --- a/llvm/include/llvm/ADT/BitmaskEnum.h +++ b/llvm/include/llvm/ADT/BitmaskEnum.h @@ -14,6 +14,7 @@ #include <utility> #include "llvm/ADT/STLForwardCompat.h" +#include "llvm/ADT/bit.h" #include "llvm/Support/MathExtras.h" /// LLVM_MARK_AS_BITMASK_ENUM lets you opt in an individual enum type so you can @@ -138,10 +139,6 @@ template <typename E> constexpr std::underlying_type_t<E> Underlying(E Val) { return U; } -constexpr unsigned bitWidth(uint64_t Value) { - return Value ? 1 + bitWidth(Value >> 1) : 0; -} - template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>> constexpr bool operator!(E Val) { return Val == static_cast<E>(0); @@ -220,7 +217,7 @@ e &operator>>=(e &lhs, e rhs) { // Enable bitmask enums in namespace ::llvm and all nested namespaces. LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>> -constexpr unsigned BitWidth = BitmaskEnumDetail::bitWidth( +constexpr unsigned BitWidth = llvm::bit_width_constexpr( uint64_t{llvm::to_underlying(E::LLVM_BITMASK_LARGEST_ENUMERATOR)}); } // namespace llvm diff --git a/llvm/include/llvm/ADT/ConcurrentHashtable.h b/llvm/include/llvm/ADT/ConcurrentHashtable.h index 6de194d..6a943c5 100644 --- a/llvm/include/llvm/ADT/ConcurrentHashtable.h +++ b/llvm/include/llvm/ADT/ConcurrentHashtable.h @@ -253,9 +253,8 @@ public: OS << "\nOverall number of entries = " << OverallNumberOfEntries; OS << "\nOverall number of non empty buckets = " << NumberOfNonEmptyBuckets; - for (auto &BucketSize : BucketSizesMap) - OS << "\n Number of buckets with size " << BucketSize.first << ": " - << BucketSize.second; + for (auto [Size, Count] : BucketSizesMap) + OS << "\n Number of buckets with size " << Size << ": " << Count; std::stringstream stream; stream << std::fixed << std::setprecision(2) diff --git a/llvm/include/llvm/ADT/DirectedGraph.h b/llvm/include/llvm/ADT/DirectedGraph.h index 83c0bea..fb6b180 100644 --- a/llvm/include/llvm/ADT/DirectedGraph.h +++ b/llvm/include/llvm/ADT/DirectedGraph.h @@ -181,16 +181,6 @@ public: DirectedGraph() = default; explicit DirectedGraph(NodeType &N) : Nodes() { addNode(N); } - DirectedGraph(const DGraphType &G) : Nodes(G.Nodes) {} - DirectedGraph(DGraphType &&RHS) : Nodes(std::move(RHS.Nodes)) {} - DGraphType &operator=(const DGraphType &G) { - Nodes = G.Nodes; - return *this; - } - DGraphType &operator=(const DGraphType &&G) { - Nodes = std::move(G.Nodes); - return *this; - } const_iterator begin() const { return Nodes.begin(); } const_iterator end() const { return Nodes.end(); } diff --git a/llvm/include/llvm/ADT/PointerUnion.h b/llvm/include/llvm/ADT/PointerUnion.h index ca0e1ed..7b66177 100644 --- a/llvm/include/llvm/ADT/PointerUnion.h +++ b/llvm/include/llvm/ADT/PointerUnion.h @@ -31,7 +31,7 @@ namespace pointer_union_detail { /// Determine the number of bits required to store integers with values < n. /// This is ceil(log2(n)). constexpr int bitsRequired(unsigned n) { - return n > 1 ? 1 + bitsRequired((n + 1) / 2) : 0; + return n == 0 ? 0 : llvm::bit_width_constexpr(n - 1); } template <typename... Ts> constexpr int lowBitsAvailable() { diff --git a/llvm/include/llvm/ADT/bit.h b/llvm/include/llvm/ADT/bit.h index 67c0a1c..66c4f94 100644 --- a/llvm/include/llvm/ADT/bit.h +++ b/llvm/include/llvm/ADT/bit.h @@ -292,6 +292,23 @@ template <typename T> [[nodiscard]] int bit_width(T Value) { return std::numeric_limits<T>::digits - llvm::countl_zero(Value); } +/// Returns the number of bits needed to represent Value if Value is nonzero. +/// Returns 0 otherwise. +/// +/// A constexpr version of bit_width. +/// +/// Ex. bit_width_constexpr(5) == 3. +template <typename T> [[nodiscard]] constexpr int bit_width_constexpr(T Value) { + static_assert(std::is_unsigned_v<T>, + "Only unsigned integral types are allowed."); + int Width = 0; + while (Value > 0) { + Value >>= 1; + ++Width; + } + return Width; +} + /// Returns the largest integral power of two no greater than Value if Value is /// nonzero. Returns 0 otherwise. /// diff --git a/llvm/include/llvm/Analysis/IR2Vec.h b/llvm/include/llvm/Analysis/IR2Vec.h index b7c3015..ed43f19 100644 --- a/llvm/include/llvm/Analysis/IR2Vec.h +++ b/llvm/include/llvm/Analysis/IR2Vec.h @@ -210,6 +210,13 @@ public: const_iterator end() const { return const_iterator(this, getNumSections(), 0); } + + using VocabMap = std::map<std::string, Embedding>; + /// Parse a vocabulary section from JSON and populate the target vocabulary + /// map. + static Error parseVocabSection(StringRef Key, + const json::Value &ParsedVocabValue, + VocabMap &TargetVocab, unsigned &Dim); }; /// Class for storing and accessing the IR2Vec vocabulary. @@ -600,8 +607,6 @@ class IR2VecVocabAnalysis : public AnalysisInfoMixin<IR2VecVocabAnalysis> { Error readVocabulary(VocabMap &OpcVocab, VocabMap &TypeVocab, VocabMap &ArgVocab); - Error parseVocabSection(StringRef Key, const json::Value &ParsedVocabValue, - VocabMap &TargetVocab, unsigned &Dim); void generateVocabStorage(VocabMap &OpcVocab, VocabMap &TypeVocab, VocabMap &ArgVocab); void emitError(Error Err, LLVMContext &Ctx); diff --git a/llvm/include/llvm/Analysis/MemoryProfileInfo.h b/llvm/include/llvm/Analysis/MemoryProfileInfo.h index be690a4..571caf9 100644 --- a/llvm/include/llvm/Analysis/MemoryProfileInfo.h +++ b/llvm/include/llvm/Analysis/MemoryProfileInfo.h @@ -59,14 +59,6 @@ LLVM_ABI std::string getAllocTypeAttributeString(AllocationType Type); /// True if the AllocTypes bitmask contains just a single type. LLVM_ABI bool hasSingleAllocType(uint8_t AllocTypes); -/// Removes any existing "ambiguous" memprof attribute. Called before we apply a -/// specific allocation type such as "cold", "notcold", or "hot". -LLVM_ABI void removeAnyExistingAmbiguousAttribute(CallBase *CB); - -/// Adds an "ambiguous" memprof attribute to call with a matched allocation -/// profile but that we haven't yet been able to disambiguate. -LLVM_ABI void addAmbiguousAttribute(CallBase *CB); - /// Class to build a trie of call stack contexts for a particular profiled /// allocation call, along with their associated allocation types. /// The allocation will be at the root of the trie, which is then used to diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h index 15ff129..af218ba 100644 --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -613,6 +613,12 @@ LLVM_ABI bool isValidAssumeForContext(const Instruction *I, const DominatorTree *DT = nullptr, bool AllowEphemerals = false); +/// Returns true, if no instruction between \p Assume and \p CtxI may free +/// memory and the function is marked as NoSync. The latter ensures the current +/// function cannot arrange for another thread to free on its behalf. +LLVM_ABI bool willNotFreeBetween(const Instruction *Assume, + const Instruction *CtxI); + enum class OverflowResult { /// Always overflows in the direction of signed/unsigned min value. AlwaysOverflowsLow, diff --git a/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/llvm/include/llvm/CodeGen/MachineFrameInfo.h index b37c677..50ce931 100644 --- a/llvm/include/llvm/CodeGen/MachineFrameInfo.h +++ b/llvm/include/llvm/CodeGen/MachineFrameInfo.h @@ -501,8 +501,12 @@ public: StackID == TargetStackID::ScalablePredicateVector; } - bool isScalableStackID(int ObjectIdx) const { + bool hasScalableStackID(int ObjectIdx) const { uint8_t StackID = getStackID(ObjectIdx); + return isScalableStackID(StackID); + } + + bool isScalableStackID(uint8_t StackID) const { return StackID == TargetStackID::ScalableVector || StackID == TargetStackID::ScalablePredicateVector; } diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index d9d6f0b..62c0806 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1959,6 +1959,10 @@ public: LLVM_ABI SDValue makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, SDValue NewMemOp); + /// Get all the nodes in their topological order without modifying any states. + LLVM_ABI void getTopologicallyOrderedNodes( + SmallVectorImpl<const SDNode *> &SortedNodes) const; + /// Topological-sort the AllNodes list and a /// assign a unique node id for each node in the DAG based on their /// topological order. Returns the number of nodes. @@ -2009,7 +2013,9 @@ public: /// function mirrors \c llvm::salvageDebugInfo. LLVM_ABI void salvageDebugInfo(SDNode &N); - LLVM_ABI void dump() const; + /// Dump the textual format of this DAG. Print nodes in sorted orders if \p + /// Sorted is true. + LLVM_ABI void dump(bool Sorted = false) const; /// In most cases this function returns the ABI alignment for a given type, /// except for illegal vector types where the alignment exceeds that of the diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 7bbad17..88691b9 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -4654,23 +4654,6 @@ public: return false; } - /// Allows the target to handle physreg-carried dependency - /// in target-specific way. Used from the ScheduleDAGSDNodes to decide whether - /// to add the edge to the dependency graph. - /// Def - input: Selection DAG node defininfg physical register - /// User - input: Selection DAG node using physical register - /// Op - input: Number of User operand - /// PhysReg - inout: set to the physical register if the edge is - /// necessary, unchanged otherwise - /// Cost - inout: physical register copy cost. - /// Returns 'true' is the edge is necessary, 'false' otherwise - virtual bool checkForPhysRegDependency(SDNode *Def, SDNode *User, unsigned Op, - const TargetRegisterInfo *TRI, - const TargetInstrInfo *TII, - MCRegister &PhysReg, int &Cost) const { - return false; - } - /// Target-specific combining of register parts into its original value virtual SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h index bf133f0..822245f 100644 --- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h +++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h @@ -109,10 +109,15 @@ public: return MC->contains(Reg1.asMCReg(), Reg2.asMCReg()); } - /// Return the cost of copying a value between two registers in this class. - /// A negative number means the register class is very expensive - /// to copy e.g. status flag register classes. - int getCopyCost() const { return MC->getCopyCost(); } + /// Return the cost of copying a value between two registers in this class. If + /// this is the maximum value, the register may be impossible to copy. + uint8_t getCopyCost() const { return MC->getCopyCost(); } + + /// \return true if register class is very expensive to copy e.g. status flag + /// register classes. + bool expensiveOrImpossibleToCopy() const { + return MC->getCopyCost() == std::numeric_limits<uint8_t>::max(); + } /// Return true if this register class may be used to create virtual /// registers. diff --git a/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFDataExtractorSimple.h b/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFDataExtractorSimple.h index 52af205..ffe0b50 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFDataExtractorSimple.h +++ b/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFDataExtractorSimple.h @@ -179,6 +179,7 @@ public: class DWARFDataExtractorSimple : public DWARFDataExtractorBase<DWARFDataExtractorSimple> { +public: using DWARFDataExtractorBase::DWARFDataExtractorBase; LLVM_ABI uint64_t getRelocatedValueImpl(uint32_t Size, uint64_t *Off, diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td index 38f95a1..bba0d6e 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMP.td +++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td @@ -1333,6 +1333,9 @@ def OMP_Tile : Directive<[Spelling<"tile">]> { let allowedOnceClauses = [ VersionedClause<OMPC_Sizes, 51>, ]; + let requiredClauses = [ + VersionedClause<OMPC_Sizes, 51>, + ]; let association = AS_Loop; let category = CA_Executable; } diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index 0a11617..5331cb5 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -4001,15 +4001,17 @@ public: /// Keeps track of value of iteration variable for input/scan loop to be /// used for Scan directive lowering - llvm::Value *IV; + llvm::Value *IV = nullptr; /// Stores the span of canonical loop being lowered to be used for temporary /// buffer allocation or Finalization. - llvm::Value *Span; + llvm::Value *Span = nullptr; ScanInfo() { ScanBuffPtrs = new llvm::SmallDenseMap<llvm::Value *, llvm::Value *>(); } + ScanInfo(ScanInfo &) = delete; + ScanInfo &operator=(const ScanInfo &) = delete; ~ScanInfo() { delete (ScanBuffPtrs); } }; diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index eb0440f..0622bfa 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -810,6 +810,26 @@ public: /// Whether the intrinsic is signed or unsigned. bool isSigned() const { return isSigned(getIntrinsicID()); }; + /// Whether the intrinsic is a smin or umin. + static bool isMin(Intrinsic::ID ID) { + switch (ID) { + case Intrinsic::umin: + case Intrinsic::smin: + return true; + case Intrinsic::umax: + case Intrinsic::smax: + return false; + default: + llvm_unreachable("Invalid intrinsic"); + } + } + + /// Whether the intrinsic is a smin or a umin. + bool isMin() const { return isMin(getIntrinsicID()); } + + /// Whether the intrinsic is a smax or a umax. + bool isMax() const { return !isMin(getIntrinsicID()); } + /// Min/max intrinsics are monotonic, they operate on a fixed-bitwidth values, /// so there is a certain threshold value, upon reaching which, /// their value can no longer change. Return said threshold. diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td index fbc92d7..b0269ee 100644 --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -162,7 +162,7 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<ArgIndex<1>>]>; class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], @@ -187,13 +187,13 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". class AdvSIMD_3VectorArg_Scalar_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<ArgIndex<2>>]>; class AdvSIMD_CvtFxToFP_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<ArgIndex<1>>]>; class AdvSIMD_CvtFPToFx_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<ArgIndex<1>>]>; class AdvSIMD_1Arg_Intrinsic : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>; @@ -221,7 +221,7 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". // Arithmetic ops -let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in { +let TargetPrefix = "aarch64" in { // Vector Add Across Lanes def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 6168e24..2e31fe5 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -2773,6 +2773,14 @@ m_MaskedLoad(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2, return m_Intrinsic<Intrinsic::masked_load>(Op0, Op1, Op2, Op3); } +/// Matches MaskedStore Intrinsic. +template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3> +inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty +m_MaskedStore(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2, + const Opnd3 &Op3) { + return m_Intrinsic<Intrinsic::masked_store>(Op0, Op1, Op2, Op3); +} + /// Matches MaskedGather Intrinsic. template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3> inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty diff --git a/llvm/include/llvm/IR/ValueMap.h b/llvm/include/llvm/IR/ValueMap.h index 97653c2..9ab7d8b 100644 --- a/llvm/include/llvm/IR/ValueMap.h +++ b/llvm/include/llvm/IR/ValueMap.h @@ -44,8 +44,8 @@ namespace llvm { template <typename KeyT, typename ValueT, typename Config> class ValueMapCallbackVH; -template <typename DenseMapT, typename KeyT> class ValueMapIterator; -template <typename DenseMapT, typename KeyT> class ValueMapConstIterator; +template <typename DenseMapT, typename KeyT, bool IsConst> +class ValueMapIteratorImpl; /// This class defines the default behavior for configurable aspects of /// ValueMap<>. User Configs should inherit from this class to be as compatible @@ -132,8 +132,8 @@ public: return Where->second.get(); } - using iterator = ValueMapIterator<MapT, KeyT>; - using const_iterator = ValueMapConstIterator<MapT, KeyT>; + using iterator = ValueMapIteratorImpl<MapT, KeyT, false>; + using const_iterator = ValueMapIteratorImpl<MapT, KeyT, true>; inline iterator begin() { return iterator(Map.begin()); } inline iterator end() { return iterator(Map.end()); } @@ -318,8 +318,10 @@ struct DenseMapInfo<ValueMapCallbackVH<KeyT, ValueT, Config>> { } }; -template <typename DenseMapT, typename KeyT> class ValueMapIterator { - using BaseT = typename DenseMapT::iterator; +template <typename DenseMapT, typename KeyT, bool IsConst> +class ValueMapIteratorImpl { + using BaseT = std::conditional_t<IsConst, typename DenseMapT::const_iterator, + typename DenseMapT::iterator>; using ValueT = typename DenseMapT::mapped_type; BaseT I; @@ -331,14 +333,20 @@ public: using pointer = value_type *; using reference = value_type &; - ValueMapIterator() : I() {} - ValueMapIterator(BaseT I) : I(I) {} + ValueMapIteratorImpl() = default; + ValueMapIteratorImpl(BaseT I) : I(I) {} + + // Allow conversion from iterator to const_iterator. + template <bool C = IsConst, typename = std::enable_if_t<C>> + ValueMapIteratorImpl( + const ValueMapIteratorImpl<DenseMapT, KeyT, false> &Other) + : I(Other.base()) {} BaseT base() const { return I; } struct ValueTypeProxy { const KeyT first; - ValueT &second; + std::conditional_t<IsConst, const ValueT &, ValueT &> second; ValueTypeProxy *operator->() { return this; } @@ -354,69 +362,25 @@ public: ValueTypeProxy operator->() const { return operator*(); } - bool operator==(const ValueMapIterator &RHS) const { return I == RHS.I; } - bool operator!=(const ValueMapIterator &RHS) const { return I != RHS.I; } + bool operator==(const ValueMapIteratorImpl &RHS) const { return I == RHS.I; } + bool operator!=(const ValueMapIteratorImpl &RHS) const { return I != RHS.I; } - inline ValueMapIterator &operator++() { // Preincrement + inline ValueMapIteratorImpl &operator++() { // Preincrement ++I; return *this; } - ValueMapIterator operator++(int) { // Postincrement - ValueMapIterator tmp = *this; + ValueMapIteratorImpl operator++(int) { // Postincrement + ValueMapIteratorImpl tmp = *this; ++*this; return tmp; } }; -template <typename DenseMapT, typename KeyT> class ValueMapConstIterator { - using BaseT = typename DenseMapT::const_iterator; - using ValueT = typename DenseMapT::mapped_type; - - BaseT I; - -public: - using iterator_category = std::forward_iterator_tag; - using value_type = std::pair<KeyT, typename DenseMapT::mapped_type>; - using difference_type = std::ptrdiff_t; - using pointer = value_type *; - using reference = value_type &; - - ValueMapConstIterator() : I() {} - ValueMapConstIterator(BaseT I) : I(I) {} - ValueMapConstIterator(ValueMapIterator<DenseMapT, KeyT> Other) - : I(Other.base()) {} - - BaseT base() const { return I; } +template <typename DenseMapT, typename KeyT> +using ValueMapIterator = ValueMapIteratorImpl<DenseMapT, KeyT, false>; - struct ValueTypeProxy { - const KeyT first; - const ValueT &second; - ValueTypeProxy *operator->() { return this; } - operator std::pair<KeyT, ValueT>() const { - return std::make_pair(first, second); - } - }; - - ValueTypeProxy operator*() const { - ValueTypeProxy Result = {I->first.Unwrap(), I->second}; - return Result; - } - - ValueTypeProxy operator->() const { return operator*(); } - - bool operator==(const ValueMapConstIterator &RHS) const { return I == RHS.I; } - bool operator!=(const ValueMapConstIterator &RHS) const { return I != RHS.I; } - - inline ValueMapConstIterator &operator++() { // Preincrement - ++I; - return *this; - } - ValueMapConstIterator operator++(int) { // Postincrement - ValueMapConstIterator tmp = *this; - ++*this; - return tmp; - } -}; +template <typename DenseMapT, typename KeyT> +using ValueMapConstIterator = ValueMapIteratorImpl<DenseMapT, KeyT, true>; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCRegisterInfo.h b/llvm/include/llvm/MC/MCRegisterInfo.h index aad3792..e6fc707 100644 --- a/llvm/include/llvm/MC/MCRegisterInfo.h +++ b/llvm/include/llvm/MC/MCRegisterInfo.h @@ -45,7 +45,7 @@ public: const uint16_t RegSetSize; const uint16_t ID; const uint16_t RegSizeInBits; - const int8_t CopyCost; + const uint8_t CopyCost; const bool Allocatable; const bool BaseClass; @@ -94,7 +94,7 @@ public: /// getCopyCost - Return the cost of copying a value between two registers in /// this class. A negative number means the register class is very expensive /// to copy e.g. status flag register classes. - int getCopyCost() const { return CopyCost; } + uint8_t getCopyCost() const { return CopyCost; } /// isAllocatable - Return true if this register class may be used to create /// virtual registers. diff --git a/llvm/include/llvm/Support/Jobserver.h b/llvm/include/llvm/Support/Jobserver.h new file mode 100644 index 0000000..6bee3b5 --- /dev/null +++ b/llvm/include/llvm/Support/Jobserver.h @@ -0,0 +1,162 @@ +//===- llvm/Support/Jobserver.h - Jobserver Client --------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines a client for the GNU Make jobserver protocol. This allows +// LLVM tools to coordinate parallel execution with a parent `make` process. +// +// The jobserver protocol is a mechanism for GNU Make to share its pool of +// available "job slots" with the subprocesses it invokes. This is particularly +// useful for tools that can perform parallel operations themselves (e.g., a +// multi-threaded linker or compiler). By participating in this protocol, a +// tool can ensure the total number of concurrent jobs does not exceed the +// limit specified by the user (e.g., `make -j8`). +// +// How it works: +// +// 1. Establishment: +// A child process discovers the jobserver by inspecting the `MAKEFLAGS` +// environment variable. If a jobserver is active, this variable will +// contain a `--jobserver-auth=<value>` argument. The format of `<value>` +// determines how to communicate with the server. +// +// 2. The Implicit Slot: +// Every command invoked by `make` is granted one "implicit" job slot. This +// means a tool can always perform at least one unit of work without needing +// to communicate with the jobserver. This implicit slot should NEVER be +// released back to the jobserver. +// +// 3. Acquiring and Releasing Slots: +// On POSIX systems, the jobserver is implemented as a pipe. The +// `--jobserver-auth` value specifies either a path to a named pipe +// (`fifo:PATH`) or a pair of file descriptors (`R,W`). The pipe is +// pre-loaded with single-character tokens, one for each available job slot. +// +// - To acquire an additional slot, a client reads a single-character token +// from the pipe. +// - To release a slot, the client must write the *exact same* character +// token back to the pipe. +// +// It is critical that a client releases all acquired slots before it exits, +// even in cases of error, to avoid deadlocking the build. +// +// Example: +// A multi-threaded linker invoked by `make -j8` wants to use multiple +// threads. It first checks for the jobserver. It knows it has one implicit +// slot, so it can use one thread. It then tries to acquire 7 more slots by +// reading 7 tokens from the jobserver pipe. If it only receives 3 tokens, +// it knows it can use a total of 1 (implicit) + 3 (acquired) = 4 threads. +// Before exiting, it must write the 3 tokens it read back to the pipe. +// +// For more context, see: +// - GNU Make manual on job slots: +// https://www.gnu.org/software/make/manual/html_node/Job-Slots.html +// - LLVM RFC discussion on jobserver support: +// https://discourse.llvm.org/t/rfc-adding-gnu-make-jobserver- +// support-to-llvm-for-coordinated-parallelism/87034 +// - Ninja’s jobserver support PR: +// https://github.com/ninja-build/ninja/pull/2506 +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_JOBSERVER_H +#define LLVM_SUPPORT_JOBSERVER_H + +#include "llvm/ADT/StringRef.h" +#include <memory> +#include <string> + +namespace llvm { + +/// A JobSlot represents a single job slot that can be acquired from or released +/// to a jobserver pool. This class is move-only. +class JobSlot { +public: + /// Default constructor creates an invalid instance. + JobSlot() = default; + + // Move operations are allowed. + JobSlot(JobSlot &&Other) noexcept : Value(Other.Value) { + Other.Value = kInvalidValue; + } + JobSlot &operator=(JobSlot &&Other) noexcept { + if (this != &Other) { + this->Value = Other.Value; + Other.Value = kInvalidValue; + } + return *this; + } + + // Copy operations are disallowed. + JobSlot(const JobSlot &) = delete; + JobSlot &operator=(const JobSlot &) = delete; + + /// Returns true if this instance is valid (either implicit or explicit). + bool isValid() const { return Value >= 0; } + + /// Returns true if this instance represents the implicit job slot. + bool isImplicit() const { return Value == kImplicitValue; } + + static JobSlot createExplicit(uint8_t V) { + return JobSlot(static_cast<int16_t>(V)); + } + + static JobSlot createImplicit() { return JobSlot(kImplicitValue); } + + uint8_t getExplicitValue() const; + bool isExplicit() const { return isValid() && !isImplicit(); } + +private: + friend class JobserverClient; + friend class JobserverClientImpl; + + JobSlot(int16_t V) : Value(V) {} + + /// The jobserver pipe carries explicit tokens (bytes 0–255). We reserve two + /// sentinels in Value for special cases: + /// kInvalidValue (-1): no slot held + /// kImplicitValue (INT16_MAX): implicit slot granted at startup (no pipe + /// I/O) + /// + /// We use int16_t so Value can store 0–255 explicit tokens and + /// sentinels without overflow, enforces fixed 16-bit width, and avoids + /// unsigned/signed mix-ups. + static constexpr int16_t kInvalidValue = -1; + static constexpr int16_t kImplicitValue = INT16_MAX; + int16_t Value = kInvalidValue; +}; + +/// The public interface for a jobserver client. +/// This client is a lazy-initialized singleton that is created on first use. +class JobserverClient { +public: + virtual ~JobserverClient(); + + /// Tries to acquire a job slot from the pool. On failure (e.g., if the pool + /// is empty), this returns an invalid JobSlot instance. The first successful + /// call will always return the implicit slot. + virtual JobSlot tryAcquire() = 0; + + /// Releases a job slot back to the pool. + virtual void release(JobSlot Slot) = 0; + + /// Returns the number of job slots available, as determined on first use. + /// This value is cached. Returns 0 if no jobserver is active. + virtual unsigned getNumJobs() const = 0; + + /// Returns the singleton instance of the JobserverClient. + /// The instance is created on the first call to this function. + /// Returns a nullptr if no jobserver is configured or an error occurs. + static JobserverClient *getInstance(); + + /// Resets the singleton instance. For testing purposes only. + static void resetForTesting(); +}; + +} // end namespace llvm + +#endif // LLVM_SUPPORT_JOBSERVER_H diff --git a/llvm/include/llvm/Support/ScopedPrinter.h b/llvm/include/llvm/Support/ScopedPrinter.h index a08cc8f..94080e8 100644 --- a/llvm/include/llvm/Support/ScopedPrinter.h +++ b/llvm/include/llvm/Support/ScopedPrinter.h @@ -284,9 +284,11 @@ public: startLine() << Label << ": " << (Value ? "Yes" : "No") << '\n'; } - template <typename... T> void printVersion(StringRef Label, T... Version) { + template <typename T, typename... TArgs> + void printVersion(StringRef Label, T MajorVersion, TArgs... MinorVersions) { startLine() << Label << ": "; - printVersionInternal(Version...); + getOStream() << MajorVersion; + ((getOStream() << '.' << MinorVersions), ...); getOStream() << "\n"; } @@ -454,16 +456,6 @@ public: virtual raw_ostream &getOStream() { return OS; } private: - template <typename T> void printVersionInternal(T Value) { - getOStream() << Value; - } - - template <typename S, typename T, typename... TArgs> - void printVersionInternal(S Value, T Value2, TArgs... Args) { - getOStream() << Value << "."; - printVersionInternal(Value2, Args...); - } - static bool flagName(const FlagEntry &LHS, const FlagEntry &RHS) { return LHS.Name < RHS.Name; } diff --git a/llvm/include/llvm/Support/ThreadPool.h b/llvm/include/llvm/Support/ThreadPool.h index c26681c..c20efc7 100644 --- a/llvm/include/llvm/Support/ThreadPool.h +++ b/llvm/include/llvm/Support/ThreadPool.h @@ -16,6 +16,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/Config/llvm-config.h" #include "llvm/Support/Compiler.h" +#include "llvm/Support/Jobserver.h" #include "llvm/Support/RWMutex.h" #include "llvm/Support/Threading.h" #include "llvm/Support/thread.h" @@ -180,6 +181,7 @@ private: void grow(int requested); void processTasks(ThreadPoolTaskGroup *WaitingForGroup); + void processTasksWithJobserver(); /// Threads in flight std::vector<llvm::thread> Threads; @@ -208,6 +210,8 @@ private: /// Maximum number of threads to potentially grow this pool to. const unsigned MaxThreadCount; + + JobserverClient *TheJobserver = nullptr; }; #endif // LLVM_ENABLE_THREADS diff --git a/llvm/include/llvm/Support/Threading.h b/llvm/include/llvm/Support/Threading.h index d3fe0a5..8884680 100644 --- a/llvm/include/llvm/Support/Threading.h +++ b/llvm/include/llvm/Support/Threading.h @@ -142,6 +142,11 @@ constexpr bool llvm_is_multithreaded() { return LLVM_ENABLE_THREADS; } /// the thread shall remain on the actual CPU socket. LLVM_ABI std::optional<unsigned> compute_cpu_socket(unsigned ThreadPoolNum) const; + + /// If true, the thread pool will attempt to coordinate with a GNU Make + /// jobserver, acquiring a job slot before processing a task. If no + /// jobserver is found in the environment, this is ignored. + bool UseJobserver = false; }; /// Build a strategy from a number of threads as a string provided in \p Num. @@ -210,6 +215,19 @@ constexpr bool llvm_is_multithreaded() { return LLVM_ENABLE_THREADS; } return S; } + /// Returns a thread strategy that attempts to coordinate with a GNU Make + /// jobserver. The number of active threads will be limited by the number of + /// available job slots. If no jobserver is detected in the environment, this + /// strategy falls back to the default hardware_concurrency() behavior. + inline ThreadPoolStrategy jobserver_concurrency() { + ThreadPoolStrategy S; + S.UseJobserver = true; + // We can still request all threads be created, as they will simply + // block waiting for a job slot if the jobserver is the limiting factor. + S.ThreadsRequested = 0; // 0 means 'use all available' + return S; + } + /// Return the current thread id, as used in various OS system calls. /// Note that not all platforms guarantee that the value returned will be /// unique across the entire system, so portable code should not assume diff --git a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h index 1e07fbe..faaff4a 100644 --- a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h +++ b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h @@ -18,8 +18,7 @@ #include "llvm/Support/DataTypes.h" -namespace llvm { -namespace X86Disassembler { +namespace llvm::X86Disassembler { #define INSTRUCTIONS_SYM x86DisassemblerInstrSpecifiers #define CONTEXTS_SYM x86DisassemblerContexts @@ -541,7 +540,6 @@ static const unsigned X86_MAX_OPERANDS = 6; /// respectively. enum DisassemblerMode { MODE_16BIT, MODE_32BIT, MODE_64BIT }; -} // namespace X86Disassembler -} // namespace llvm +} // namespace llvm::X86Disassembler #endif diff --git a/llvm/include/llvm/TableGen/CodeGenHelpers.h b/llvm/include/llvm/TableGen/CodeGenHelpers.h new file mode 100644 index 0000000..7dca6a0 --- /dev/null +++ b/llvm/include/llvm/TableGen/CodeGenHelpers.h @@ -0,0 +1,67 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines common utilities for generating C++ code. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TABLEGEN_CODEGENHELPERS_H +#define LLVM_TABLEGEN_CODEGENHELPERS_H + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/raw_ostream.h" +#include <string> + +namespace llvm { +// Simple RAII helper for emitting ifdef-undef-endif scope. +class IfDefEmitter { +public: + IfDefEmitter(raw_ostream &OS, StringRef Name) : Name(Name.str()), OS(OS) { + OS << "#ifdef " << Name << "\n" + << "#undef " << Name << "\n\n"; + } + ~IfDefEmitter() { OS << "\n#endif // " << Name << "\n\n"; } + +private: + std::string Name; + raw_ostream &OS; +}; + +// Simple RAII helper for emitting namespace scope. Name can be a single +// namespace (empty for anonymous namespace) or nested namespace. +class NamespaceEmitter { +public: + NamespaceEmitter(raw_ostream &OS, StringRef Name) : OS(OS) { + emitNamespaceStarts(Name); + } + + ~NamespaceEmitter() { close(); } + + // Explicit function to close the namespace scopes. + void close() { + for (StringRef NS : llvm::reverse(Namespaces)) + OS << "} // namespace " << NS << "\n"; + Namespaces.clear(); + } + +private: + void emitNamespaceStarts(StringRef Name) { + llvm::SplitString(Name, Namespaces, "::"); + for (StringRef NS : Namespaces) + OS << "namespace " << NS << " {\n"; + } + + SmallVector<StringRef, 2> Namespaces; + raw_ostream &OS; +}; + +} // end namespace llvm + +#endif // LLVM_TABLEGEN_CODEGENHELPERS_H diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index faf7788..e3f995d 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -126,7 +126,7 @@ def G_FRAME_INDEX : GenericInstruction { } def G_GLOBAL_VALUE : GenericInstruction { - let OutOperandList = (outs type0:$dst); + let OutOperandList = (outs ptype0:$dst); let InOperandList = (ins unknown:$src); let hasSideEffects = false; } diff --git a/llvm/include/llvm/Target/TargetSchedule.td b/llvm/include/llvm/Target/TargetSchedule.td index f55bff1..d6a4059 100644 --- a/llvm/include/llvm/Target/TargetSchedule.td +++ b/llvm/include/llvm/Target/TargetSchedule.td @@ -377,6 +377,12 @@ class MCSchedPredicate<MCInstPredicate P> : SchedPredicateBase { SchedMachineModel SchedModel = ?; } +// A scheduling predicate whose logic depends on a SubtargetFeature. +class FeatureSchedPredicate<SubtargetFeature SF> : SchedPredicateBase { + SubtargetFeature Feature = SF; + SchedMachineModel SchedModel = ?; +} + // Define a predicate to determine which SchedVariant applies to a // particular MachineInstr. The code snippet is used as an // if-statement's expression. Available variables are MI, SchedModel, diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h index 2454149..74a4d6c 100644 --- a/llvm/include/llvm/Transforms/Scalar/GVN.h +++ b/llvm/include/llvm/Transforms/Scalar/GVN.h @@ -56,6 +56,7 @@ class OptimizationRemarkEmitter; class PHINode; class TargetLibraryInfo; class Value; +class IntrinsicInst; /// A private "module" namespace for types and utilities used by GVN. These /// are implementation details and should not be used by clients. namespace LLVM_LIBRARY_VISIBILITY_NAMESPACE gvn { @@ -349,6 +350,7 @@ private: // Helper functions of redundant load elimination. bool processLoad(LoadInst *L); + bool processMaskedLoad(IntrinsicInst *I); bool processNonLocalLoad(LoadInst *L); bool processAssumeIntrinsic(AssumeInst *II); |