diff options
Diffstat (limited to 'llvm/include')
23 files changed, 420 insertions, 131 deletions
diff --git a/llvm/include/llvm/ADT/AllocatorList.h b/llvm/include/llvm/ADT/AllocatorList.h index 04d0afc..2716b83 100644 --- a/llvm/include/llvm/ADT/AllocatorList.h +++ b/llvm/include/llvm/ADT/AllocatorList.h @@ -155,8 +155,8 @@ public: std::swap(getAlloc(), RHS.getAlloc()); } - bool empty() { return List.empty(); } - size_t size() { return List.size(); } + [[nodiscard]] bool empty() const { return List.empty(); } + [[nodiscard]] size_t size() const { return List.size(); } iterator begin() { return iterator(List.begin()); } iterator end() { return iterator(List.end()); } diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h index fb91690..448d100 100644 --- a/llvm/include/llvm/ADT/ArrayRef.h +++ b/llvm/include/llvm/ADT/ArrayRef.h @@ -547,7 +547,8 @@ namespace llvm { } template <typename T> - inline bool operator==(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) { + [[nodiscard]] inline bool operator==(const SmallVectorImpl<T> &LHS, + ArrayRef<T> RHS) { return ArrayRef<T>(LHS).equals(RHS); } @@ -557,7 +558,8 @@ namespace llvm { } template <typename T> - inline bool operator!=(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) { + [[nodiscard]] inline bool operator!=(const SmallVectorImpl<T> &LHS, + ArrayRef<T> RHS) { return !(LHS == RHS); } diff --git a/llvm/include/llvm/ADT/DenseMap.h b/llvm/include/llvm/ADT/DenseMap.h index bcf3e96..4bda50f 100644 --- a/llvm/include/llvm/ADT/DenseMap.h +++ b/llvm/include/llvm/ADT/DenseMap.h @@ -440,10 +440,6 @@ protected: } } - static unsigned getHashValue(const KeyT &Val) { - return KeyInfoT::getHashValue(Val); - } - template <typename LookupKeyT> static unsigned getHashValue(const LookupKeyT &Val) { return KeyInfoT::getHashValue(Val); diff --git a/llvm/include/llvm/ADT/EquivalenceClasses.h b/llvm/include/llvm/ADT/EquivalenceClasses.h index 1a2331c..90d8948 100644 --- a/llvm/include/llvm/ADT/EquivalenceClasses.h +++ b/llvm/include/llvm/ADT/EquivalenceClasses.h @@ -180,8 +180,8 @@ public: } /// Returns true if \p V is contained an equivalence class. - bool contains(const ElemTy &V) const { - return TheMapping.find(V) != TheMapping.end(); + [[nodiscard]] bool contains(const ElemTy &V) const { + return TheMapping.contains(V); } /// getLeaderValue - Return the leader for the specified value that is in the @@ -256,9 +256,11 @@ public: } if (!Next) { // If the current element is the last element(not leader), set the - // successor of the current element's predecessor to null, and set - // the 'Leader' field of the class leader to the predecessor element. - Pre->Next = nullptr; + // successor of the current element's predecessor to null while + // preserving the leader bit, and set the 'Leader' field of the class + // leader to the predecessor element. + Pre->Next = reinterpret_cast<const ECValue *>( + static_cast<intptr_t>(Pre->isLeader())); Leader->Leader = Pre; } else { // If the current element is in the middle of class, then simply diff --git a/llvm/include/llvm/ADT/GenericUniformityImpl.h b/llvm/include/llvm/ADT/GenericUniformityImpl.h index 141816c3..7fb0dbe 100644 --- a/llvm/include/llvm/ADT/GenericUniformityImpl.h +++ b/llvm/include/llvm/ADT/GenericUniformityImpl.h @@ -408,15 +408,6 @@ public: const CycleT *); protected: - /// \brief Value/block pair representing a single phi input. - struct PhiInput { - ConstValueRefT value; - BlockT *predBlock; - - PhiInput(ConstValueRefT value, BlockT *predBlock) - : value(value), predBlock(predBlock) {} - }; - const ContextT &Context; const FunctionT &F; const CycleInfoT &CI; diff --git a/llvm/include/llvm/ADT/PackedVector.h b/llvm/include/llvm/ADT/PackedVector.h index 77fcbf2..09c20e3 100644 --- a/llvm/include/llvm/ADT/PackedVector.h +++ b/llvm/include/llvm/ADT/PackedVector.h @@ -20,53 +20,6 @@ namespace llvm { -template <typename T, unsigned BitNum, typename BitVectorTy, bool isSigned> -class PackedVectorBase; - -// This won't be necessary if we can specialize members without specializing -// the parent template. -template <typename T, unsigned BitNum, typename BitVectorTy> -class PackedVectorBase<T, BitNum, BitVectorTy, false> { -protected: - static T getValue(const BitVectorTy &Bits, unsigned Idx) { - T val = T(); - for (unsigned i = 0; i != BitNum; ++i) - val = T(val | ((Bits[(Idx * BitNum) + i] ? 1UL : 0UL) << i)); - return val; - } - - static void setValue(BitVectorTy &Bits, unsigned Idx, T val) { - assert((val >> BitNum) == 0 && "value is too big"); - for (unsigned i = 0; i != BitNum; ++i) - Bits[(Idx * BitNum) + i] = val & (T(1) << i); - } -}; - -template <typename T, unsigned BitNum, typename BitVectorTy> -class PackedVectorBase<T, BitNum, BitVectorTy, true> { -protected: - static T getValue(const BitVectorTy &Bits, unsigned Idx) { - T val = T(); - for (unsigned i = 0; i != BitNum - 1; ++i) - val = T(val | ((Bits[(Idx * BitNum) + i] ? 1UL : 0UL) << i)); - if (Bits[(Idx * BitNum) + BitNum - 1]) - val = ~val; - return val; - } - - static void setValue(BitVectorTy &Bits, unsigned Idx, T val) { - if (val < 0) { - val = ~val; - Bits.set((Idx * BitNum) + BitNum - 1); - } else { - Bits.reset((Idx * BitNum) + BitNum - 1); - } - assert((val >> (BitNum - 1)) == 0 && "value is too big"); - for (unsigned i = 0; i != BitNum - 1; ++i) - Bits[(Idx * BitNum) + i] = val & (T(1) << i); - } -}; - /// Store a vector of values using a specific number of bits for each /// value. Both signed and unsigned types can be used, e.g /// @code @@ -75,16 +28,46 @@ protected: /// will create a vector accepting values -2, -1, 0, 1. Any other value will hit /// an assertion. template <typename T, unsigned BitNum, typename BitVectorTy = BitVector> -class PackedVector - : public PackedVectorBase<T, BitNum, BitVectorTy, - std::numeric_limits<T>::is_signed> { +class PackedVector { BitVectorTy Bits; // Keep track of the number of elements on our own. // We always maintain Bits.size() == NumElements * BitNum. // Used to avoid an integer division in size(). unsigned NumElements = 0; - using base = PackedVectorBase<T, BitNum, BitVectorTy, - std::numeric_limits<T>::is_signed>; + + static T getValue(const BitVectorTy &Bits, unsigned Idx) { + if constexpr (std::numeric_limits<T>::is_signed) { + T val = T(); + for (unsigned i = 0; i != BitNum - 1; ++i) + val = T(val | ((Bits[(Idx * BitNum) + i] ? 1UL : 0UL) << i)); + if (Bits[(Idx * BitNum) + BitNum - 1]) + val = ~val; + return val; + } else { + T val = T(); + for (unsigned i = 0; i != BitNum; ++i) + val = T(val | ((Bits[(Idx * BitNum) + i] ? 1UL : 0UL) << i)); + return val; + } + } + + static void setValue(BitVectorTy &Bits, unsigned Idx, T val) { + if constexpr (std::numeric_limits<T>::is_signed) { + if (val < 0) { + val = ~val; + Bits.set((Idx * BitNum) + BitNum - 1); + } else { + Bits.reset((Idx * BitNum) + BitNum - 1); + } + assert((val >> (BitNum - 1)) == 0 && "value is too big"); + for (unsigned i = 0; i != BitNum - 1; ++i) + Bits[(Idx * BitNum) + i] = val & (T(1) << i); + } else { + assert((val >> BitNum) == 0 && "value is too big"); + for (unsigned i = 0; i != BitNum; ++i) + Bits[(Idx * BitNum) + i] = val & (T(1) << i); + } + } public: class reference { @@ -135,7 +118,7 @@ public: reference operator[](unsigned Idx) { return reference(*this, Idx); } - T operator[](unsigned Idx) const { return base::getValue(Bits, Idx); } + T operator[](unsigned Idx) const { return getValue(Bits, Idx); } bool operator==(const PackedVector &RHS) const { return Bits == RHS.Bits; } diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h index 52ab385..84b4ad7 100644 --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -724,8 +724,9 @@ public: /// Return true if the block BB needs to be predicated in order for the loop /// to be vectorized. - LLVM_ABI static bool blockNeedsPredication(BasicBlock *BB, Loop *TheLoop, - DominatorTree *DT); + LLVM_ABI static bool blockNeedsPredication(const BasicBlock *BB, + const Loop *TheLoop, + const DominatorTree *DT); /// Returns true if value \p V is loop invariant. LLVM_ABI bool isInvariant(Value *V) const; diff --git a/llvm/include/llvm/CAS/FileOffset.h b/llvm/include/llvm/CAS/FileOffset.h new file mode 100644 index 0000000..21d045e --- /dev/null +++ b/llvm/include/llvm/CAS/FileOffset.h @@ -0,0 +1,39 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// This file declares interface for FileOffset that represent stored data at an +/// offset from the beginning of a file. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CAS_FILEOFFSET_H +#define LLVM_CAS_FILEOFFSET_H + +#include <cstdlib> + +namespace llvm::cas { + +/// FileOffset is a wrapper around `uint64_t` to represent the offset of data +/// from the beginning of the file. +class FileOffset { +public: + uint64_t get() const { return Offset; } + + explicit operator bool() const { return Offset; } + + FileOffset() = default; + explicit FileOffset(uint64_t Offset) : Offset(Offset) {} + +private: + uint64_t Offset = 0; +}; + +} // namespace llvm::cas + +#endif // LLVM_CAS_FILEOFFSET_H diff --git a/llvm/include/llvm/CAS/OnDiskTrieRawHashMap.h b/llvm/include/llvm/CAS/OnDiskTrieRawHashMap.h new file mode 100644 index 0000000..5e41bf6 --- /dev/null +++ b/llvm/include/llvm/CAS/OnDiskTrieRawHashMap.h @@ -0,0 +1,236 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// This file declares interface for OnDiskTrieRawHashMap, a thread-safe and +/// (mostly) lock-free hash map stored as trie and backed by persistent files on +/// disk. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CAS_ONDISKTRIERAWHASHMAP_H +#define LLVM_CAS_ONDISKTRIERAWHASHMAP_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/CAS/FileOffset.h" +#include "llvm/Support/Error.h" +#include <optional> + +namespace llvm { + +class raw_ostream; + +namespace cas { + +/// OnDiskTrieRawHashMap is a persistent trie data structure used as hash maps. +/// The keys are fixed length, and are expected to be binary hashes with a +/// normal distribution. +/// +/// - Thread-safety is achieved through the use of atomics within a shared +/// memory mapping. Atomic access does not work on networked filesystems. +/// - Filesystem locks are used, but only sparingly: +/// - during initialization, for creating / opening an existing store; +/// - for the lifetime of the instance, a shared/reader lock is held +/// - during destruction, if there are no concurrent readers, to shrink the +/// files to their minimum size. +/// - Path is used as a directory: +/// - "index" stores the root trie and subtries. +/// - "data" stores (most of) the entries, like a bump-ptr-allocator. +/// - Large entries are stored externally in a file named by the key. +/// - Code is system-dependent and binary format itself is not portable. These +/// are not artifacts that can/should be moved between different systems; they +/// are only appropriate for local storage. +class OnDiskTrieRawHashMap { +public: + LLVM_DUMP_METHOD void dump() const; + void + print(raw_ostream &OS, + function_ref<void(ArrayRef<char>)> PrintRecordData = nullptr) const; + +public: + /// Const value proxy to access the records stored in TrieRawHashMap. + struct ConstValueProxy { + ConstValueProxy() = default; + ConstValueProxy(ArrayRef<uint8_t> Hash, ArrayRef<char> Data) + : Hash(Hash), Data(Data) {} + ConstValueProxy(ArrayRef<uint8_t> Hash, StringRef Data) + : Hash(Hash), Data(Data.begin(), Data.size()) {} + + ArrayRef<uint8_t> Hash; + ArrayRef<char> Data; + }; + + /// Value proxy to access the records stored in TrieRawHashMap. + struct ValueProxy { + operator ConstValueProxy() const { return ConstValueProxy(Hash, Data); } + + ValueProxy() = default; + ValueProxy(ArrayRef<uint8_t> Hash, MutableArrayRef<char> Data) + : Hash(Hash), Data(Data) {} + + ArrayRef<uint8_t> Hash; + MutableArrayRef<char> Data; + }; + + /// Validate the trie data structure. + /// + /// Callback receives the file offset to the data entry and the data stored. + Error validate( + function_ref<Error(FileOffset, ConstValueProxy)> RecordVerifier) const; + + /// Check the valid range of file offset for OnDiskTrieRawHashMap. + static bool validOffset(FileOffset Offset) { + return Offset.get() < (1LL << 48); + } + +public: + /// Template class to implement a `pointer` type into the trie data structure. + /// + /// It provides pointer-like operation, e.g., dereference to get underlying + /// data. It also reserves the top 16 bits of the pointer value, which can be + /// used to pack additional information if needed. + template <class ProxyT> class PointerImpl { + public: + FileOffset getOffset() const { + return FileOffset(OffsetLow32 | (uint64_t)OffsetHigh16 << 32); + } + + explicit operator bool() const { return IsValue; } + + const ProxyT &operator*() const { + assert(IsValue); + return Value; + } + const ProxyT *operator->() const { + assert(IsValue); + return &Value; + } + + PointerImpl() = default; + + protected: + PointerImpl(ProxyT Value, FileOffset Offset, bool IsValue = true) + : Value(Value), OffsetLow32((uint64_t)Offset.get()), + OffsetHigh16((uint64_t)Offset.get() >> 32), IsValue(IsValue) { + if (IsValue) + assert(validOffset(Offset)); + } + + ProxyT Value; + uint32_t OffsetLow32 = 0; + uint16_t OffsetHigh16 = 0; + + // True if points to a value (not a "nullptr"). Use an extra field because + // 0 can be a valid offset. + bool IsValue = false; + }; + + class pointer; + class const_pointer : public PointerImpl<ConstValueProxy> { + public: + const_pointer() = default; + + private: + friend class pointer; + friend class OnDiskTrieRawHashMap; + using const_pointer::PointerImpl::PointerImpl; + }; + + class pointer : public PointerImpl<ValueProxy> { + public: + operator const_pointer() const { + return const_pointer(Value, getOffset(), IsValue); + } + + pointer() = default; + + private: + friend class OnDiskTrieRawHashMap; + using pointer::PointerImpl::PointerImpl; + }; + + /// Find the value from hash. + /// + /// \returns pointer to the value if exists, otherwise returns a non-value + /// pointer that evaluates to `false` when convert to boolean. + const_pointer find(ArrayRef<uint8_t> Hash) const; + + /// Helper function to recover a pointer into the trie from file offset. + Expected<const_pointer> recoverFromFileOffset(FileOffset Offset) const; + + using LazyInsertOnConstructCB = + function_ref<void(FileOffset TentativeOffset, ValueProxy TentativeValue)>; + using LazyInsertOnLeakCB = + function_ref<void(FileOffset TentativeOffset, ValueProxy TentativeValue, + FileOffset FinalOffset, ValueProxy FinalValue)>; + + /// Insert lazily. + /// + /// \p OnConstruct is called when ready to insert a value, after allocating + /// space for the data. It is called at most once. + /// + /// \p OnLeak is called only if \p OnConstruct has been called and a race + /// occurred before insertion, causing the tentative offset and data to be + /// abandoned. This allows clients to clean up other results or update any + /// references. + /// + /// NOTE: Does *not* guarantee that \p OnConstruct is only called on success. + /// The in-memory \a TrieRawHashMap uses LazyAtomicPointer to synchronize + /// simultaneous writes, but that seems dangerous to use in a memory-mapped + /// file in case a process crashes in the busy state. + Expected<pointer> insertLazy(ArrayRef<uint8_t> Hash, + LazyInsertOnConstructCB OnConstruct = nullptr, + LazyInsertOnLeakCB OnLeak = nullptr); + + Expected<pointer> insert(const ConstValueProxy &Value) { + return insertLazy(Value.Hash, [&](FileOffset, ValueProxy Allocated) { + assert(Allocated.Hash == Value.Hash); + assert(Allocated.Data.size() == Value.Data.size()); + llvm::copy(Value.Data, Allocated.Data.begin()); + }); + } + + size_t size() const; + size_t capacity() const; + + /// Gets or creates a file at \p Path with a hash-mapped trie named \p + /// TrieName. The hash size is \p NumHashBits (in bits) and the records store + /// data of size \p DataSize (in bytes). + /// + /// \p MaxFileSize controls the maximum file size to support, limiting the + /// size of the \a mapped_file_region. \p NewFileInitialSize is the starting + /// size if a new file is created. + /// + /// \p NewTableNumRootBits and \p NewTableNumSubtrieBits are hints to + /// configure the trie, if it doesn't already exist. + /// + /// \pre NumHashBits is a multiple of 8 (byte-aligned). + static Expected<OnDiskTrieRawHashMap> + create(const Twine &Path, const Twine &TrieName, size_t NumHashBits, + uint64_t DataSize, uint64_t MaxFileSize, + std::optional<uint64_t> NewFileInitialSize, + std::optional<size_t> NewTableNumRootBits = std::nullopt, + std::optional<size_t> NewTableNumSubtrieBits = std::nullopt); + + OnDiskTrieRawHashMap(OnDiskTrieRawHashMap &&RHS); + OnDiskTrieRawHashMap &operator=(OnDiskTrieRawHashMap &&RHS); + ~OnDiskTrieRawHashMap(); + +private: + struct ImplType; + explicit OnDiskTrieRawHashMap(std::unique_ptr<ImplType> Impl); + std::unique_ptr<ImplType> Impl; +}; + +} // namespace cas +} // namespace llvm + +#endif // LLVM_CAS_ONDISKTRIERAWHASHMAP_H diff --git a/llvm/include/llvm/CodeGen/DebugHandlerBase.h b/llvm/include/llvm/CodeGen/DebugHandlerBase.h index 2849497..fee4bb1 100644 --- a/llvm/include/llvm/CodeGen/DebugHandlerBase.h +++ b/llvm/include/llvm/CodeGen/DebugHandlerBase.h @@ -144,6 +144,8 @@ public: static bool isUnsignedDIType(const DIType *Ty); const InstructionOrdering &getInstOrdering() const { return InstOrdering; } + + const LexicalScopes &getLexicalScopes() const { return LScopes; } }; } // namespace llvm diff --git a/llvm/include/llvm/CodeGen/LexicalScopes.h b/llvm/include/llvm/CodeGen/LexicalScopes.h index 4172e90..993df54 100644 --- a/llvm/include/llvm/CodeGen/LexicalScopes.h +++ b/llvm/include/llvm/CodeGen/LexicalScopes.h @@ -141,12 +141,18 @@ class LexicalScopes { public: LexicalScopes() = default; + /// Scan module to build subprogram-to-function map. + LLVM_ABI void initialize(const Module &); + /// Scan machine function and constuct lexical scope nest, resets /// the instance if necessary. - LLVM_ABI void initialize(const MachineFunction &); + LLVM_ABI void scanFunction(const MachineFunction &); + + /// Reset the instance so that it's prepared for another module. + LLVM_ABI void resetModule(); - /// Release memory. - LLVM_ABI void reset(); + /// Reset the instance so that it's prepared for another function. + LLVM_ABI void resetFunction(); /// Return true if there is any lexical scope information available. bool empty() { return CurrentFnLexicalScope == nullptr; } @@ -196,6 +202,11 @@ public: /// Find or create an abstract lexical scope. LLVM_ABI LexicalScope *getOrCreateAbstractScope(const DILocalScope *Scope); + /// Get function to which the given subprogram is attached, if exists. + const Function *getFunction(const DISubprogram *SP) const { + return FunctionMap.lookup(SP); + } + private: /// Find lexical scope for the given Scope/IA. If not available /// then create new lexical scope. @@ -225,6 +236,9 @@ private: const MachineFunction *MF = nullptr; + /// Mapping between DISubprograms and IR functions. + DenseMap<const DISubprogram *, const Function *> FunctionMap; + /// Tracks the scopes in the current function. // Use an unordered_map to ensure value pointer validity over insertion. std::unordered_map<const DILocalScope *, LexicalScope> LexicalScopeMap; diff --git a/llvm/include/llvm/CodeGen/ValueTypes.td b/llvm/include/llvm/CodeGen/ValueTypes.td index 9ea127d..300addd 100644 --- a/llvm/include/llvm/CodeGen/ValueTypes.td +++ b/llvm/include/llvm/CodeGen/ValueTypes.td @@ -367,11 +367,11 @@ def aarch64mfp8 : ValueType<8, 253>; // 8-bit value in FPR (AArch64) def c64 : VTCheriCapability<64, 254>; // 64-bit CHERI capability value def c128 : VTCheriCapability<128, 255>; // 128-bit CHERI capability value +let isNormalValueType = false in { // Pseudo valuetype mapped to the current CHERI capability pointer size. // Should only be used in TableGen. def cPTR : VTAny<503>; -let isNormalValueType = false in { def token : ValueType<0, 504>; // TokenTy def MetadataVT : ValueType<0, 505> { // Metadata let LLVMName = "Metadata"; diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td index 4af9ffc..81fbfbf 100644 --- a/llvm/include/llvm/IR/IntrinsicsX86.td +++ b/llvm/include/llvm/IR/IntrinsicsX86.td @@ -1919,62 +1919,62 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_vpdpbssd_128 : ClangBuiltin<"__builtin_ia32_vpdpbssd128">, DefaultAttrsIntrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_avx2_vpdpbssd_256 : ClangBuiltin<"__builtin_ia32_vpdpbssd256">, DefaultAttrsIntrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; def int_x86_avx2_vpdpbssds_128 : ClangBuiltin<"__builtin_ia32_vpdpbssds128">, DefaultAttrsIntrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_avx2_vpdpbssds_256 : ClangBuiltin<"__builtin_ia32_vpdpbssds256">, DefaultAttrsIntrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; def int_x86_avx2_vpdpbsud_128 : ClangBuiltin<"__builtin_ia32_vpdpbsud128">, DefaultAttrsIntrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_avx2_vpdpbsud_256 : ClangBuiltin<"__builtin_ia32_vpdpbsud256">, DefaultAttrsIntrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; def int_x86_avx2_vpdpbsuds_128 : ClangBuiltin<"__builtin_ia32_vpdpbsuds128">, DefaultAttrsIntrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_avx2_vpdpbsuds_256 : ClangBuiltin<"__builtin_ia32_vpdpbsuds256">, DefaultAttrsIntrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; def int_x86_avx2_vpdpbuud_128 : ClangBuiltin<"__builtin_ia32_vpdpbuud128">, DefaultAttrsIntrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_avx2_vpdpbuud_256 : ClangBuiltin<"__builtin_ia32_vpdpbuud256">, DefaultAttrsIntrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; def int_x86_avx2_vpdpbuuds_128 : ClangBuiltin<"__builtin_ia32_vpdpbuuds128">, DefaultAttrsIntrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_avx2_vpdpbuuds_256 : ClangBuiltin<"__builtin_ia32_vpdpbuuds256">, DefaultAttrsIntrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>; def int_x86_avx2_vpdpwsud_128 @@ -5000,32 +5000,32 @@ let TargetPrefix = "x86" in { def int_x86_avx10_vpdpbssd_512 : ClangBuiltin<"__builtin_ia32_vpdpbssd512">, DefaultAttrsIntrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; def int_x86_avx10_vpdpbssds_512 : ClangBuiltin<"__builtin_ia32_vpdpbssds512">, DefaultAttrsIntrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; def int_x86_avx10_vpdpbsud_512 : ClangBuiltin<"__builtin_ia32_vpdpbsud512">, DefaultAttrsIntrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; def int_x86_avx10_vpdpbsuds_512 : ClangBuiltin<"__builtin_ia32_vpdpbsuds512">, DefaultAttrsIntrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; def int_x86_avx10_vpdpbuud_512 : ClangBuiltin<"__builtin_ia32_vpdpbuud512">, DefaultAttrsIntrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; def int_x86_avx10_vpdpbuuds_512 : ClangBuiltin<"__builtin_ia32_vpdpbuuds512">, DefaultAttrsIntrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; // VNNI INT16 def int_x86_avx10_vpdpwsud_512 : diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h index de9675f..e97160e 100644 --- a/llvm/include/llvm/IR/ProfDataUtils.h +++ b/llvm/include/llvm/IR/ProfDataUtils.h @@ -185,6 +185,14 @@ inline uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale) { LLVM_ABI void setExplicitlyUnknownBranchWeights(Instruction &I, StringRef PassName); +/// Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch +/// weights in the new instruction if the parent function of the original +/// instruction has an entry count. This is to not confuse users by injecting +/// profile data into non-profiled functions. +LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, + Function &F, + StringRef PassName); + /// Analogous to setExplicitlyUnknownBranchWeights, but for functions and their /// entry counts. LLVM_ABI void setExplicitlyUnknownFunctionEntryCount(Function &F, diff --git a/llvm/include/llvm/Object/OffloadBundle.h b/llvm/include/llvm/Object/OffloadBundle.h index f4d5a1d..18be62b 100644 --- a/llvm/include/llvm/Object/OffloadBundle.h +++ b/llvm/include/llvm/Object/OffloadBundle.h @@ -161,7 +161,7 @@ public: OffsetStr.getAsInteger(10, O); Str = Str.drop_front(OffsetStr.size()); - if (Str.consume_front("&size=")) + if (!Str.consume_front("&size=")) return createStringError(object_error::parse_failed, "Reading 'size' in URI"); diff --git a/llvm/include/llvm/Support/FileCollector.h b/llvm/include/llvm/Support/FileCollector.h index 9cc6776..9fa11ba 100644 --- a/llvm/include/llvm/Support/FileCollector.h +++ b/llvm/include/llvm/Support/FileCollector.h @@ -81,6 +81,9 @@ public: /// Canonicalize a pair of virtual and real paths. LLVM_ABI PathStorage canonicalize(StringRef SrcPath); + /// Return the underlying file system. + vfs::FileSystem &getFileSystem() const { return *VFS; }; + explicit PathCanonicalizer(IntrusiveRefCntPtr<vfs::FileSystem> VFS) : VFS(std::move(VFS)) {} diff --git a/llvm/include/llvm/Support/Mustache.h b/llvm/include/llvm/Support/Mustache.h index 781ec55..ee9f406 100644 --- a/llvm/include/llvm/Support/Mustache.h +++ b/llvm/include/llvm/Support/Mustache.h @@ -85,6 +85,14 @@ using SectionLambda = std::function<llvm::json::Value(std::string)>; class ASTNode; using AstPtr = std::unique_ptr<ASTNode>; +using EscapeMap = DenseMap<char, std::string>; + +struct MustacheContext { + StringMap<AstPtr> Partials; + StringMap<Lambda> Lambdas; + StringMap<SectionLambda> SectionLambdas; + EscapeMap Escapes; +}; // A Template represents the container for the AST and the partials // and Lambdas that are registered with it. @@ -118,10 +126,7 @@ public: LLVM_ABI void overrideEscapeCharacters(DenseMap<char, std::string> Escapes); private: - StringMap<AstPtr> Partials; - StringMap<Lambda> Lambdas; - StringMap<SectionLambda> SectionLambdas; - DenseMap<char, std::string> Escapes; + MustacheContext Ctx; AstPtr Tree; }; } // namespace llvm::mustache diff --git a/llvm/include/llvm/Support/SipHash.h b/llvm/include/llvm/Support/SipHash.h index 910cf594..b090565 100644 --- a/llvm/include/llvm/Support/SipHash.h +++ b/llvm/include/llvm/Support/SipHash.h @@ -33,6 +33,13 @@ LLVM_ABI void getSipHash_2_4_64(ArrayRef<uint8_t> In, const uint8_t (&K)[16], LLVM_ABI void getSipHash_2_4_128(ArrayRef<uint8_t> In, const uint8_t (&K)[16], uint8_t (&Out)[16]); +/// Compute a stable 64-bit hash of the given string. +/// +/// The exact algorithm is the little-endian interpretation of the +/// non-doubled (i.e. 64-bit) result of applying a SipHash-2-4 using +/// a specific seed value which can be found in the source. +LLVM_ABI uint64_t getStableSipHash(StringRef Str); + /// Compute a stable non-zero 16-bit hash of the given string. /// /// The exact algorithm is the little-endian interpretation of the diff --git a/llvm/include/llvm/Support/TrailingObjects.h b/llvm/include/llvm/Support/TrailingObjects.h index 3eb7c0b..dc03285 100644 --- a/llvm/include/llvm/Support/TrailingObjects.h +++ b/llvm/include/llvm/Support/TrailingObjects.h @@ -284,11 +284,8 @@ public: /// (which must be one of those specified in the class template). The /// array may have zero or more elements in it. template <typename T> T *getTrailingObjects() { - verifyTrailingObjectsAssertions<true>(); - // Forwards to an impl function with overloads, since member - // function templates can't be specialized. - return this->getTrailingObjectsImpl( - static_cast<BaseTy *>(this), TrailingObjectsBase::OverloadToken<T>()); + return const_cast<T *>( + static_cast<const TrailingObjects *>(this)->getTrailingObjects<T>()); } // getTrailingObjects() specialization for a single trailing type. @@ -306,13 +303,8 @@ public: } FirstTrailingType *getTrailingObjects() { - static_assert(sizeof...(TrailingTys) == 1, - "Can use non-templated getTrailingObjects() only when there " - "is a single trailing type"); - verifyTrailingObjectsAssertions<false>(); - return this->getTrailingObjectsImpl( - static_cast<BaseTy *>(this), - TrailingObjectsBase::OverloadToken<FirstTrailingType>()); + return const_cast<FirstTrailingType *>( + static_cast<const TrailingObjects *>(this)->getTrailingObjects()); } // Functions that return the trailing objects as ArrayRefs. @@ -342,9 +334,8 @@ public: } template <typename T> T *getTrailingObjectsNonStrict() { - verifyTrailingObjectsAssertions<false>(); - return this->getTrailingObjectsImpl( - static_cast<BaseTy *>(this), TrailingObjectsBase::OverloadToken<T>()); + return const_cast<T *>(static_cast<const TrailingObjects *>(this) + ->getTrailingObjectsNonStrict<T>()); } template <typename T> diff --git a/llvm/include/llvm/Support/TypeSize.h b/llvm/include/llvm/Support/TypeSize.h index 29d1c68..0a7ae15 100644 --- a/llvm/include/llvm/Support/TypeSize.h +++ b/llvm/include/llvm/Support/TypeSize.h @@ -179,7 +179,7 @@ public: /// This function tells the caller whether the element count is known at /// compile time to be a multiple of the scalar value RHS. constexpr bool isKnownMultipleOf(ScalarTy RHS) const { - return getKnownMinValue() % RHS == 0; + return RHS != 0 && getKnownMinValue() % RHS == 0; } /// Returns whether or not the callee is known to be a multiple of RHS. @@ -191,7 +191,8 @@ public: // x % y == 0 !=> x % (vscale * y) == 0 if (!isScalable() && RHS.isScalable()) return false; - return getKnownMinValue() % RHS.getKnownMinValue() == 0; + return RHS.getKnownMinValue() != 0 && + getKnownMinValue() % RHS.getKnownMinValue() == 0; } // Return the minimum value with the assumption that the count is exact. diff --git a/llvm/include/llvm/TextAPI/SymbolSet.h b/llvm/include/llvm/TextAPI/SymbolSet.h index 42c411a..22f4124 100644 --- a/llvm/include/llvm/TextAPI/SymbolSet.h +++ b/llvm/include/llvm/TextAPI/SymbolSet.h @@ -92,6 +92,8 @@ private: public: SymbolSet() = default; + SymbolSet(const SymbolSet &other) = delete; + SymbolSet &operator=(const SymbolSet &other) = delete; LLVM_ABI ~SymbolSet(); LLVM_ABI Symbol *addGlobal(EncodeKind Kind, StringRef Name, SymbolFlags Flags, const Target &Targ); diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h index fa313f5..d6c2d7f 100644 --- a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h +++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h @@ -64,6 +64,8 @@ protected: /// A worklist of the instructions that need to be simplified. InstructionWorklist &Worklist; + Function &F; + // Mode in which we are running the combiner. const bool MinimizeSize; @@ -98,17 +100,17 @@ protected: bool ComputedBackEdges = false; public: - InstCombiner(InstructionWorklist &Worklist, BuilderTy &Builder, - bool MinimizeSize, AAResults *AA, AssumptionCache &AC, - TargetLibraryInfo &TLI, TargetTransformInfo &TTI, - DominatorTree &DT, OptimizationRemarkEmitter &ORE, - BlockFrequencyInfo *BFI, BranchProbabilityInfo *BPI, - ProfileSummaryInfo *PSI, const DataLayout &DL, + InstCombiner(InstructionWorklist &Worklist, BuilderTy &Builder, Function &F, + AAResults *AA, AssumptionCache &AC, TargetLibraryInfo &TLI, + TargetTransformInfo &TTI, DominatorTree &DT, + OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI, + BranchProbabilityInfo *BPI, ProfileSummaryInfo *PSI, + const DataLayout &DL, ReversePostOrderTraversal<BasicBlock *> &RPOT) : TTIForTargetIntrinsicsOnly(TTI), Builder(Builder), Worklist(Worklist), - MinimizeSize(MinimizeSize), AA(AA), AC(AC), TLI(TLI), DT(DT), DL(DL), - SQ(DL, &TLI, &DT, &AC, nullptr, /*UseInstrInfo*/ true, - /*CanUseUndef*/ true, &DC), + F(F), MinimizeSize(F.hasMinSize()), AA(AA), AC(AC), TLI(TLI), DT(DT), + DL(DL), SQ(DL, &TLI, &DT, &AC, nullptr, /*UseInstrInfo*/ true, + /*CanUseUndef*/ true, &DC), ORE(ORE), BFI(BFI), BPI(BPI), PSI(PSI), RPOT(RPOT) {} virtual ~InstCombiner() = default; diff --git a/llvm/include/llvm/Transforms/Instrumentation/GCOVProfiler.h b/llvm/include/llvm/Transforms/Instrumentation/GCOVProfiler.h index 4d3ead29..f53cee2 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/GCOVProfiler.h +++ b/llvm/include/llvm/Transforms/Instrumentation/GCOVProfiler.h @@ -20,11 +20,15 @@ namespace llvm { /// The gcov-style instrumentation pass class GCOVProfilerPass : public PassInfoMixin<GCOVProfilerPass> { public: - GCOVProfilerPass(const GCOVOptions &Options = GCOVOptions::getDefault()) : GCOVOpts(Options) { } + GCOVProfilerPass( + const GCOVOptions &Options = GCOVOptions::getDefault(), + IntrusiveRefCntPtr<vfs::FileSystem> VFS = vfs::getRealFileSystem()) + : GCOVOpts(Options), VFS(std::move(VFS)) {} LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: GCOVOptions GCOVOpts; + IntrusiveRefCntPtr<vfs::FileSystem> VFS; }; } // namespace llvm |