diff options
Diffstat (limited to 'llvm/include')
62 files changed, 782 insertions, 181 deletions
diff --git a/llvm/include/llvm/ADT/IndexedMap.h b/llvm/include/llvm/ADT/IndexedMap.h index 55935a7..02193c7 100644 --- a/llvm/include/llvm/ADT/IndexedMap.h +++ b/llvm/include/llvm/ADT/IndexedMap.h @@ -43,40 +43,40 @@ class IndexedMap { // is trivially copyable. using StorageT = SmallVector<T, 0>; - StorageT storage_; - T nullVal_ = T(); - ToIndexT toIndex_; + StorageT Storage; + T NullVal = T(); + ToIndexT ToIndex; public: IndexedMap() = default; - explicit IndexedMap(const T &val) : nullVal_(val) {} + explicit IndexedMap(const T &Val) : NullVal(Val) {} - typename StorageT::reference operator[](IndexT n) { - assert(toIndex_(n) < storage_.size() && "index out of bounds!"); - return storage_[toIndex_(n)]; + typename StorageT::reference operator[](IndexT N) { + assert(ToIndex(N) < Storage.size() && "index out of bounds!"); + return Storage[ToIndex(N)]; } - typename StorageT::const_reference operator[](IndexT n) const { - assert(toIndex_(n) < storage_.size() && "index out of bounds!"); - return storage_[toIndex_(n)]; + typename StorageT::const_reference operator[](IndexT N) const { + assert(ToIndex(N) < Storage.size() && "index out of bounds!"); + return Storage[ToIndex(N)]; } - void reserve(typename StorageT::size_type s) { storage_.reserve(s); } + void reserve(typename StorageT::size_type S) { Storage.reserve(S); } - void resize(typename StorageT::size_type s) { storage_.resize(s, nullVal_); } + void resize(typename StorageT::size_type S) { Storage.resize(S, NullVal); } - void clear() { storage_.clear(); } + void clear() { Storage.clear(); } - void grow(IndexT n) { - unsigned NewSize = toIndex_(n) + 1; - if (NewSize > storage_.size()) + void grow(IndexT N) { + unsigned NewSize = ToIndex(N) + 1; + if (NewSize > Storage.size()) resize(NewSize); } - bool inBounds(IndexT n) const { return toIndex_(n) < storage_.size(); } + bool inBounds(IndexT N) const { return ToIndex(N) < Storage.size(); } - typename StorageT::size_type size() const { return storage_.size(); } + typename StorageT::size_type size() const { return Storage.size(); } }; } // namespace llvm diff --git a/llvm/include/llvm/ADT/RadixTree.h b/llvm/include/llvm/ADT/RadixTree.h new file mode 100644 index 0000000..a65acdd --- /dev/null +++ b/llvm/include/llvm/ADT/RadixTree.h @@ -0,0 +1,351 @@ +//===-- llvm/ADT/RadixTree.h - Radix Tree implementation --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +//===----------------------------------------------------------------------===// +// +// This file implements a Radix Tree. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_RADIXTREE_H +#define LLVM_ADT_RADIXTREE_H + +#include "llvm/ADT/ADL.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/iterator.h" +#include "llvm/ADT/iterator_range.h" +#include <cassert> +#include <cstddef> +#include <iterator> +#include <limits> +#include <list> +#include <utility> +#include <vector> + +namespace llvm { + +/// \brief A Radix Tree implementation. +/// +/// A Radix Tree (also known as a compact prefix tree or radix trie) is a +/// data structure that stores a dynamic set or associative array where keys +/// are strings and values are associated with these keys. Unlike a regular +/// trie, the edges of a radix tree can be labeled with sequences of characters +/// as well as single characters. This makes radix trees more efficient for +/// storing sparse data sets, where many nodes in a regular trie would have +/// only one child. +/// +/// This implementation supports arbitrary key types that can be iterated over +/// (e.g., `std::string`, `std::vector<char>`, `ArrayRef<char>`). The key type +/// must provide `begin()` and `end()` for iteration. +/// +/// The tree stores `std::pair<const KeyType, T>` as its value type. +/// +/// Example usage: +/// \code +/// llvm::RadixTree<StringRef, int> Tree; +/// Tree.emplace("apple", 1); +/// Tree.emplace("grapefruit", 2); +/// Tree.emplace("grape", 3); +/// +/// // Find prefixes +/// for (const auto &[Key, Value] : Tree.find_prefixes("grapefruit juice")) { +/// // pair will be {"grape", 3} +/// // pair will be {"grapefruit", 2} +/// llvm::outs() << Key << ": " << Value << "\n"; +/// } +/// +/// // Iterate over all elements +/// for (const auto &[Key, Value] : Tree) +/// llvm::outs() << Key << ": " << Value << "\n"; +/// \endcode +/// +/// \note +/// The `RadixTree` takes ownership of the `KeyType` and `T` objects +/// inserted into it. When an element is removed or the tree is destroyed, +/// these objects will be destructed. +/// However, if `KeyType` is a reference-like type, e.g., StringRef or range, +/// the user must guarantee that the referenced data has a lifetime longer than +/// the tree. +template <typename KeyType, typename T> class RadixTree { +public: + using key_type = KeyType; + using mapped_type = T; + using value_type = std::pair<const KeyType, mapped_type>; + +private: + using KeyConstIteratorType = + decltype(adl_begin(std::declval<const key_type &>())); + using KeyConstIteratorRangeType = iterator_range<KeyConstIteratorType>; + using KeyValueType = + remove_cvref_t<decltype(*adl_begin(std::declval<key_type &>()))>; + using ContainerType = std::list<value_type>; + + /// Represents an internal node in the Radix Tree. + struct Node { + KeyConstIteratorRangeType Key{KeyConstIteratorType{}, + KeyConstIteratorType{}}; + std::vector<Node> Children; + + /// An iterator to the value associated with this node. + /// + /// If this node does not have a value (i.e., it's an internal node that + /// only serves as a path to other values), this iterator will be equal + /// to default constructed `ContainerType::iterator()`. + typename ContainerType::iterator Value; + + /// The first character of the Key. Used for fast child lookup. + KeyValueType KeyFront; + + Node() = default; + Node(const KeyConstIteratorRangeType &Key) + : Key(Key), KeyFront(*Key.begin()) { + assert(!Key.empty()); + } + + Node(Node &&) = default; + Node &operator=(Node &&) = default; + + Node(const Node &) = delete; + Node &operator=(const Node &) = delete; + + const Node *findChild(const KeyConstIteratorRangeType &Key) const { + if (Key.empty()) + return nullptr; + for (const Node &Child : Children) { + assert(!Child.Key.empty()); // Only root can be empty. + if (Child.KeyFront == *Key.begin()) + return &Child; + } + return nullptr; + } + + Node *findChild(const KeyConstIteratorRangeType &Query) { + const Node *This = this; + return const_cast<Node *>(This->findChild(Query)); + } + + size_t countNodes() const { + size_t R = 1; + for (const Node &C : Children) + R += C.countNodes(); + return R; + } + + /// + /// Splits the current node into two. + /// + /// This function is used when a new key needs to be inserted that shares + /// a common prefix with the current node's key, but then diverges. + /// The current `Key` is truncated to the common prefix, and a new child + /// node is created for the remainder of the original node's `Key`. + /// + /// \param SplitPoint An iterator pointing to the character in the current + /// `Key` where the split should occur. + void split(KeyConstIteratorType SplitPoint) { + Node Child(make_range(SplitPoint, Key.end())); + Key = make_range(Key.begin(), SplitPoint); + + Children.swap(Child.Children); + std::swap(Value, Child.Value); + + Children.emplace_back(std::move(Child)); + } + }; + + /// Root always corresponds to the empty key, which is the shortest possible + /// prefix for everything. + Node Root; + ContainerType KeyValuePairs; + + /// Finds or creates a new tail or leaf node corresponding to the `Key`. + Node &findOrCreate(KeyConstIteratorRangeType Key) { + Node *Curr = &Root; + if (Key.empty()) + return *Curr; + + for (;;) { + auto [I1, I2] = llvm::mismatch(Key, Curr->Key); + Key = make_range(I1, Key.end()); + + if (I2 != Curr->Key.end()) { + // Match is partial. Either query is too short, or there is mismatching + // character. Split either way, and put new node in between of the + // current and its children. + Curr->split(I2); + + // Split was caused by mismatch, so `findChild` would fail. + break; + } + + Node *Child = Curr->findChild(Key); + if (!Child) + break; + + // Move to child with the same first character. + Curr = Child; + } + + if (Key.empty()) { + // The current node completely matches the key, return it. + return *Curr; + } + + // `Key` is a suffix of original `Key` unmatched by path from the `Root` to + // the `Curr`, and we have no candidate in the children to match more. + // Create a new one. + return Curr->Children.emplace_back(Key); + } + + /// + /// An iterator for traversing prefixes search results. + /// + /// This iterator is used by `find_prefixes` to traverse the tree and find + /// elements that are prefixes to the given key. It's a forward iterator. + /// + /// \tparam MappedType The type of the value pointed to by the iterator. + /// This will be `value_type` for non-const iterators + /// and `const value_type` for const iterators. + template <typename MappedType> + class IteratorImpl + : public iterator_facade_base<IteratorImpl<MappedType>, + std::forward_iterator_tag, MappedType> { + const Node *Curr = nullptr; + KeyConstIteratorRangeType Query{KeyConstIteratorType{}, + KeyConstIteratorType{}}; + + void findNextValid() { + while (Curr && Curr->Value == typename ContainerType::iterator()) + advance(); + } + + void advance() { + assert(Curr); + if (Query.empty()) { + Curr = nullptr; + return; + } + + Curr = Curr->findChild(Query); + if (!Curr) { + Curr = nullptr; + return; + } + + auto [I1, I2] = llvm::mismatch(Query, Curr->Key); + if (I2 != Curr->Key.end()) { + Curr = nullptr; + return; + } + Query = make_range(I1, Query.end()); + } + + friend class RadixTree; + IteratorImpl(const Node *C, const KeyConstIteratorRangeType &Q) + : Curr(C), Query(Q) { + findNextValid(); + } + + public: + IteratorImpl() = default; + + MappedType &operator*() const { return *Curr->Value; } + + IteratorImpl &operator++() { + advance(); + findNextValid(); + return *this; + } + + bool operator==(const IteratorImpl &Other) const { + return Curr == Other.Curr; + } + }; + +public: + RadixTree() = default; + RadixTree(RadixTree &&) = default; + RadixTree &operator=(RadixTree &&) = default; + + using prefix_iterator = IteratorImpl<value_type>; + using const_prefix_iterator = IteratorImpl<const value_type>; + + using iterator = typename ContainerType::iterator; + using const_iterator = typename ContainerType::const_iterator; + + /// Returns true if the tree is empty. + bool empty() const { return KeyValuePairs.empty(); } + + /// Returns the number of elements in the tree. + size_t size() const { return KeyValuePairs.size(); } + + /// Returns the number of nodes in the tree. + /// + /// This function counts all internal nodes in the tree. It can be useful for + /// understanding the memory footprint or complexity of the tree structure. + size_t countNodes() const { return Root.countNodes(); } + + /// Returns an iterator to the first element. + iterator begin() { return KeyValuePairs.begin(); } + const_iterator begin() const { return KeyValuePairs.begin(); } + + /// Returns an iterator to the end of the tree. + iterator end() { return KeyValuePairs.end(); } + const_iterator end() const { return KeyValuePairs.end(); } + + /// Constructs and inserts a new element into the tree. + /// + /// This function constructs an element in place within the tree. If an + /// element with the same key already exists, the insertion fails and the + /// function returns an iterator to the existing element along with `false`. + /// Otherwise, the new element is inserted and the function returns an + /// iterator to the new element along with `true`. + /// + /// \param Key The key of the element to construct. + /// \param Args Arguments to forward to the constructor of the mapped_type. + /// \return A pair consisting of an iterator to the inserted element (or to + /// the element that prevented insertion) and a boolean value + /// indicating whether the insertion took place. + template <typename... Ts> + std::pair<iterator, bool> emplace(key_type &&Key, Ts &&...Args) { + // We want to make new `Node` to refer key in the container, not the one + // from the argument. + // FIXME: Determine that we need a new node, before expanding + // `KeyValuePairs`. + const value_type &NewValue = KeyValuePairs.emplace_front( + std::move(Key), T(std::forward<Ts>(Args)...)); + Node &Node = findOrCreate(NewValue.first); + bool HasValue = Node.Value != typename ContainerType::iterator(); + if (!HasValue) + Node.Value = KeyValuePairs.begin(); + else + KeyValuePairs.pop_front(); + return {Node.Value, !HasValue}; + } + + /// + /// Finds all elements whose keys are prefixes of the given `Key`. + /// + /// This function returns an iterator range over all elements in the tree + /// whose keys are prefixes of the provided `Key`. For example, if the tree + /// contains "abcde", "abc", "abcdefgh", and `Key` is "abcde", this function + /// would return iterators to "abcde" and "abc". + /// + /// \param Key The key to search for prefixes of. + /// \return An `iterator_range` of `const_prefix_iterator`s, allowing + /// iteration over the found prefix elements. + /// \note The returned iterators reference the `Key` provided by the caller. + /// The caller must ensure that `Key` remains valid for the lifetime + /// of the iterators. + iterator_range<const_prefix_iterator> + find_prefixes(const key_type &Key) const { + return iterator_range<const_prefix_iterator>{ + const_prefix_iterator(&Root, KeyConstIteratorRangeType(Key)), + const_prefix_iterator{}}; + } +}; + +} // namespace llvm + +#endif // LLVM_ADT_RADIXTREE_H diff --git a/llvm/include/llvm/ADT/STLForwardCompat.h b/llvm/include/llvm/ADT/STLForwardCompat.h index 9c81981..e02694f 100644 --- a/llvm/include/llvm/ADT/STLForwardCompat.h +++ b/llvm/include/llvm/ADT/STLForwardCompat.h @@ -125,7 +125,7 @@ struct detector<std::void_t<Op<Args...>>, Op, Args...> { template <template <class...> class Op, class... Args> using is_detected = typename detail::detector<void, Op, Args...>::value_t; -struct identity_cxx20 // NOLINT(readability-identifier-naming) +struct identity // NOLINT(readability-identifier-naming) { using is_transparent = void; diff --git a/llvm/include/llvm/ADT/SmallVector.h b/llvm/include/llvm/ADT/SmallVector.h index ca0b918..51109d1 100644 --- a/llvm/include/llvm/ADT/SmallVector.h +++ b/llvm/include/llvm/ADT/SmallVector.h @@ -14,6 +14,7 @@ #ifndef LLVM_ADT_SMALLVECTOR_H #define LLVM_ADT_SMALLVECTOR_H +#include "llvm/ADT/ADL.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/Support/Compiler.h" #include <algorithm> @@ -1295,28 +1296,27 @@ inline size_t capacity_in_bytes(const SmallVector<T, N> &X) { template <typename RangeType> using ValueTypeFromRangeType = - std::remove_const_t<std::remove_reference_t<decltype(*std::begin( - std::declval<RangeType &>()))>>; + std::remove_const_t<detail::ValueOfRange<RangeType>>; /// Given a range of type R, iterate the entire range and return a /// SmallVector with elements of the vector. This is useful, for example, /// when you want to iterate a range and then sort the results. template <unsigned Size, typename R> SmallVector<ValueTypeFromRangeType<R>, Size> to_vector(R &&Range) { - return {std::begin(Range), std::end(Range)}; + return {adl_begin(Range), adl_end(Range)}; } template <typename R> SmallVector<ValueTypeFromRangeType<R>> to_vector(R &&Range) { - return {std::begin(Range), std::end(Range)}; + return {adl_begin(Range), adl_end(Range)}; } template <typename Out, unsigned Size, typename R> SmallVector<Out, Size> to_vector_of(R &&Range) { - return {std::begin(Range), std::end(Range)}; + return {adl_begin(Range), adl_end(Range)}; } template <typename Out, typename R> SmallVector<Out> to_vector_of(R &&Range) { - return {std::begin(Range), std::end(Range)}; + return {adl_begin(Range), adl_end(Range)}; } // Explicit instantiations diff --git a/llvm/include/llvm/ADT/SparseMultiSet.h b/llvm/include/llvm/ADT/SparseMultiSet.h index 5e4e170..59de4cf 100644 --- a/llvm/include/llvm/ADT/SparseMultiSet.h +++ b/llvm/include/llvm/ADT/SparseMultiSet.h @@ -82,7 +82,7 @@ namespace llvm { /// @tparam SparseT An unsigned integer type. See above. /// template <typename ValueT, typename KeyT = unsigned, - typename KeyFunctorT = identity_cxx20, typename SparseT = uint8_t> + typename KeyFunctorT = identity, typename SparseT = uint8_t> class SparseMultiSet { static_assert(std::is_unsigned_v<SparseT>, "SparseT must be an unsigned integer type"); diff --git a/llvm/include/llvm/ADT/SparseSet.h b/llvm/include/llvm/ADT/SparseSet.h index 4697de09..41fd501 100644 --- a/llvm/include/llvm/ADT/SparseSet.h +++ b/llvm/include/llvm/ADT/SparseSet.h @@ -59,24 +59,20 @@ template <typename ValueT> struct SparseSetValTraits { } }; -/// SparseSetValFunctor - Helper class for selecting SparseSetValTraits. The -/// generic implementation handles ValueT classes which either provide -/// getSparseSetIndex() or specialize SparseSetValTraits<>. +/// SparseSetValFunctor - Helper class for getting a value's index. /// +/// In the generic case, this is done via SparseSetValTraits. When the value +/// type is the same as the key type, the KeyFunctor is used directly. template <typename KeyT, typename ValueT, typename KeyFunctorT> struct SparseSetValFunctor { unsigned operator()(const ValueT &Val) const { - return SparseSetValTraits<ValueT>::getValIndex(Val); + if constexpr (std::is_same_v<KeyT, ValueT>) + return KeyFunctorT()(Val); + else + return SparseSetValTraits<ValueT>::getValIndex(Val); } }; -/// SparseSetValFunctor<KeyT, KeyT> - Helper class for the common case of -/// identity key/value sets. -template <typename KeyT, typename KeyFunctorT> -struct SparseSetValFunctor<KeyT, KeyT, KeyFunctorT> { - unsigned operator()(const KeyT &Key) const { return KeyFunctorT()(Key); } -}; - /// SparseSet - Fast set implementation for objects that can be identified by /// small unsigned keys. /// @@ -117,7 +113,7 @@ struct SparseSetValFunctor<KeyT, KeyT, KeyFunctorT> { /// @tparam SparseT An unsigned integer type. See above. /// template <typename ValueT, typename KeyT = unsigned, - typename KeyFunctorT = identity_cxx20, typename SparseT = uint8_t> + typename KeyFunctorT = identity, typename SparseT = uint8_t> class SparseSet { static_assert(std::is_unsigned_v<SparseT>, "SparseT must be an unsigned integer type"); diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h index 58793ed..2832beb 100644 --- a/llvm/include/llvm/Analysis/InstSimplifyFolder.h +++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h @@ -120,7 +120,7 @@ public: } Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Type *Ty, - Instruction *FMFSource) const override { + Instruction *FMFSource = nullptr) const override { return simplifyBinaryIntrinsic(ID, Ty, LHS, RHS, SQ, dyn_cast_if_present<CallBase>(FMFSource)); } diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 5d3b233..7b7dc1b 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -227,6 +227,9 @@ public: /// Get the kind of extension that an instruction represents. LLVM_ABI static PartialReductionExtendKind getPartialReductionExtendKind(Instruction *I); + /// Get the kind of extension that a cast opcode represents. + LLVM_ABI static PartialReductionExtendKind + getPartialReductionExtendKind(Instruction::CastOps CastOpc); /// Construct a TTI object using a type implementing the \c Concept /// API below. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 93aff35..b0601eb 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -276,7 +276,6 @@ public: SmallVector<Register> &Ops) const; /// Replace \p MI with a build_vector. - bool matchCombineShuffleToBuildVector(MachineInstr &MI) const; void applyCombineShuffleToBuildVector(MachineInstr &MI) const; /// Try to combine G_SHUFFLE_VECTOR into G_CONCAT_VECTORS. @@ -295,8 +294,6 @@ public: /// Replace \p MI with a concat_vectors with \p Ops. void applyCombineShuffleVector(MachineInstr &MI, const ArrayRef<Register> Ops) const; - bool matchShuffleToExtract(MachineInstr &MI) const; - void applyShuffleToExtract(MachineInstr &MI) const; /// Optimize memcpy intrinsics et al, e.g. constant len calls. /// /p MaxLen if non-zero specifies the max length of a mem libcall to inline. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index c0e426c..a458cbd 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -497,6 +497,7 @@ public: LLVM_ABI LegalizeResult lowerMinMax(MachineInstr &MI); LLVM_ABI LegalizeResult lowerFCopySign(MachineInstr &MI); LLVM_ABI LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI); + LLVM_ABI LegalizeResult lowerFMinimumMaximum(MachineInstr &MI); LLVM_ABI LegalizeResult lowerFMad(MachineInstr &MI); LLVM_ABI LegalizeResult lowerIntrinsicRound(MachineInstr &MI); LLVM_ABI LegalizeResult lowerFFloor(MachineInstr &MI); diff --git a/llvm/include/llvm/CodeGen/MIR2Vec.h b/llvm/include/llvm/CodeGen/MIR2Vec.h index 48bb0a5..44f009c 100644 --- a/llvm/include/llvm/CodeGen/MIR2Vec.h +++ b/llvm/include/llvm/CodeGen/MIR2Vec.h @@ -111,6 +111,11 @@ class MIRVocabulary { size_t TotalEntries = 0; } Layout; + // TODO: See if we can have only one reg classes section instead of physical + // and virtual separate sections in the vocabulary. This would reduce the + // number of vocabulary entities significantly. + // We can potentially distinguish physical and virtual registers by + // considering them as a separate feature. enum class Section : unsigned { Opcodes = 0, CommonOperands = 1, @@ -185,6 +190,25 @@ class MIRVocabulary { return Storage[static_cast<unsigned>(SectionID)][LocalIndex]; } + /// Get entity ID (flat index) for a common operand type + /// This is used for triplet generation + unsigned getEntityIDForCommonOperand( + MachineOperand::MachineOperandType OperandType) const { + return Layout.CommonOperandBase + getCommonOperandIndex(OperandType); + } + + /// Get entity ID (flat index) for a register + /// This is used for triplet generation + unsigned getEntityIDForRegister(Register Reg) const { + if (!Reg.isValid() || Reg.isStack()) + return Layout + .VirtRegBase; // Return VirtRegBase for invalid/stack registers + unsigned LocalIndex = getRegisterOperandIndex(Reg); + size_t BaseOffset = + Reg.isPhysical() ? Layout.PhyRegBase : Layout.VirtRegBase; + return BaseOffset + LocalIndex; + } + public: /// Static method for extracting base opcode names (public for testing) static std::string extractBaseOpcodeName(StringRef InstrName); @@ -201,6 +225,20 @@ public: unsigned getDimension() const { return Storage.getDimension(); } + /// Get entity ID (flat index) for an opcode + /// This is used for triplet generation + unsigned getEntityIDForOpcode(unsigned Opcode) const { + return Layout.OpcodeBase + getCanonicalOpcodeIndex(Opcode); + } + + /// Get entity ID (flat index) for a machine operand + /// This is used for triplet generation + unsigned getEntityIDForMachineOperand(const MachineOperand &MO) const { + if (MO.getType() == MachineOperand::MO_Register) + return getEntityIDForRegister(MO.getReg()); + return getEntityIDForCommonOperand(MO.getType()); + } + // Accessor methods const Embedding &operator[](unsigned Opcode) const { unsigned LocalIndex = getCanonicalOpcodeIndex(Opcode); diff --git a/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h b/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h new file mode 100644 index 0000000..d044d5f --- /dev/null +++ b/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h @@ -0,0 +1,114 @@ +//===- llvm/CodeGen/MachineBlockHashInfo.h ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Compute the hashes of basic blocks. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H +#define LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H + +#include "llvm/CodeGen/MachineFunctionPass.h" + +namespace llvm { + +/// An object wrapping several components of a basic block hash. The combined +/// (blended) hash is represented and stored as one uint64_t, while individual +/// components are of smaller size (e.g., uint16_t or uint8_t). +struct BlendedBlockHash { +public: + explicit BlendedBlockHash(uint16_t Offset, uint16_t OpcodeHash, + uint16_t InstrHash, uint16_t NeighborHash) + : Offset(Offset), OpcodeHash(OpcodeHash), InstrHash(InstrHash), + NeighborHash(NeighborHash) {} + + explicit BlendedBlockHash(uint64_t CombinedHash) { + Offset = CombinedHash & 0xffff; + CombinedHash >>= 16; + OpcodeHash = CombinedHash & 0xffff; + CombinedHash >>= 16; + InstrHash = CombinedHash & 0xffff; + CombinedHash >>= 16; + NeighborHash = CombinedHash & 0xffff; + } + + /// Combine the blended hash into uint64_t. + uint64_t combine() const { + uint64_t Hash = 0; + Hash |= uint64_t(NeighborHash); + Hash <<= 16; + Hash |= uint64_t(InstrHash); + Hash <<= 16; + Hash |= uint64_t(OpcodeHash); + Hash <<= 16; + Hash |= uint64_t(Offset); + return Hash; + } + + /// Compute a distance between two given blended hashes. The smaller the + /// distance, the more similar two blocks are. For identical basic blocks, + /// the distance is zero. + /// Since OpcodeHash is highly stable, we consider a match good only if + /// the OpcodeHashes are identical. Mismatched OpcodeHashes lead to low + /// matching accuracy, and poor matches undermine the quality of final + /// inference. Notably, during inference, we also consider the matching + /// ratio of basic blocks. For MachineFunctions with a low matching + /// ratio, we directly skip optimization to reduce the impact of + /// mismatches. This ensures even very poor profiles won’t cause negative + /// optimization. + /// In the context of matching, we consider NeighborHash to be more + /// important. This is especially true when accounting for inlining + /// scenarios, where the position of a basic block in the control + /// flow graph is more critical. + uint64_t distance(const BlendedBlockHash &BBH) const { + assert(OpcodeHash == BBH.OpcodeHash && + "incorrect blended hash distance computation"); + uint64_t Dist = 0; + // Account for NeighborHash + Dist += NeighborHash == BBH.NeighborHash ? 0 : 1; + Dist <<= 16; + // Account for InstrHash + Dist += InstrHash == BBH.InstrHash ? 0 : 1; + Dist <<= 16; + // Account for Offset + Dist += (Offset >= BBH.Offset ? Offset - BBH.Offset : BBH.Offset - Offset); + return Dist; + } + +private: + /// The offset of the basic block from the function start. + uint16_t Offset{0}; + /// Hash of the basic block instructions, excluding operands. + uint16_t OpcodeHash{0}; + /// Hash of the basic block instructions, including opcodes and + /// operands. + uint16_t InstrHash{0}; + /// OpcodeHash of the basic block together with OpcodeHashes of its + /// successors and predecessors. + uint16_t NeighborHash{0}; +}; + +class MachineBlockHashInfo : public MachineFunctionPass { + DenseMap<const MachineBasicBlock *, uint64_t> MBBHashInfo; + +public: + static char ID; + MachineBlockHashInfo(); + + StringRef getPassName() const override { return "Basic Block Hash Compute"; } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + + bool runOnMachineFunction(MachineFunction &F) override; + + uint64_t getMBBHash(const MachineBasicBlock &MBB); +}; + +} // end namespace llvm + +#endif // LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h index 7fae550..a852555 100644 --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -69,6 +69,9 @@ LLVM_ABI MachineFunctionPass *createBasicBlockSectionsPass(); LLVM_ABI MachineFunctionPass *createBasicBlockPathCloningPass(); +/// createMachineBlockHashInfoPass - This pass computes basic block hashes. +LLVM_ABI MachineFunctionPass *createMachineBlockHashInfoPass(); + /// createMachineFunctionSplitterPass - This pass splits machine functions /// using profile information. LLVM_ABI MachineFunctionPass *createMachineFunctionSplitterPass(); @@ -607,14 +610,6 @@ LLVM_ABI ModulePass *createCheckDebugMachineModulePass(); /// caller saved registers with stack slots. LLVM_ABI extern char &FixupStatepointCallerSavedID; -/// The pass transforms load/store <256 x i32> to AMX load/store intrinsics -/// or split the data to two <128 x i32>. -LLVM_ABI FunctionPass *createX86LowerAMXTypePass(); - -/// The pass transforms amx intrinsics to scalar operation if the function has -/// optnone attribute or it is O0. -LLVM_ABI FunctionPass *createX86LowerAMXIntrinsicsPass(); - /// When learning an eviction policy, extract score(reward) information, /// otherwise this does nothing LLVM_ABI FunctionPass *createRegAllocScoringPass(); diff --git a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h index 26d7080..ab0d7e3 100644 --- a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -89,7 +89,7 @@ namespace llvm { /// allocated once for the pass. It can be cleared in constant time and reused /// without any frees. using RegUnit2SUnitsMap = - SparseMultiSet<PhysRegSUOper, unsigned, identity_cxx20, uint16_t>; + SparseMultiSet<PhysRegSUOper, unsigned, identity, uint16_t>; /// Track local uses of virtual registers. These uses are gathered by the DAG /// builder and may be consulted by the scheduler to avoid iterating an entire diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 64a7563..d6ed3a8 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -2127,7 +2127,7 @@ public: /// performs validation and error handling, returns the function. Otherwise, /// returns nullptr. Must be previously inserted by insertSSPDeclarations. /// Should be used only when getIRStackGuard returns nullptr. - virtual Function *getSSPStackGuardCheck(const Module &M) const; + Function *getSSPStackGuardCheck(const Module &M) const; protected: Value *getDefaultSafeStackPointerLocation(IRBuilderBase &IRB, @@ -3576,6 +3576,10 @@ public: return nullptr; } + const RTLIB::RuntimeLibcallsInfo &getRuntimeLibcallsInfo() const { + return Libcalls; + } + void setLibcallImpl(RTLIB::Libcall Call, RTLIB::LibcallImpl Impl) { Libcalls.setLibcallImpl(Call, Impl); } @@ -3813,10 +3817,6 @@ private: /// The list of libcalls that the target will use. RTLIB::RuntimeLibcallsInfo Libcalls; - /// The ISD::CondCode that should be used to test the result of each of the - /// comparison libcall against zero. - ISD::CondCode CmpLibcallCCs[RTLIB::UNKNOWN_LIBCALL]; - /// The bits of IndexedModeActions used to store the legalisation actions /// We store the data as | ML | MS | L | S | each taking 4 bits. enum IndexedModeActionsBits { diff --git a/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h b/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h index 3a36863..5a46207 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h +++ b/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h @@ -34,7 +34,7 @@ class LLVM_ABI AppendingTypeTableBuilder : public TypeCollection { public: explicit AppendingTypeTableBuilder(BumpPtrAllocator &Storage); - ~AppendingTypeTableBuilder(); + ~AppendingTypeTableBuilder() override; // TypeCollection overrides std::optional<TypeIndex> getFirst() override; diff --git a/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h b/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h index a587b3e..93e1c99 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h +++ b/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h @@ -47,7 +47,7 @@ class LLVM_ABI GlobalTypeTableBuilder : public TypeCollection { public: explicit GlobalTypeTableBuilder(BumpPtrAllocator &Storage); - ~GlobalTypeTableBuilder(); + ~GlobalTypeTableBuilder() override; // TypeCollection overrides std::optional<TypeIndex> getFirst() override; diff --git a/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h b/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h index 7302784..b9b2669 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h +++ b/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h @@ -43,7 +43,7 @@ class LLVM_ABI MergingTypeTableBuilder : public TypeCollection { public: explicit MergingTypeTableBuilder(BumpPtrAllocator &Storage); - ~MergingTypeTableBuilder(); + ~MergingTypeTableBuilder() override; // TypeCollection overrides std::optional<TypeIndex> getFirst() override; diff --git a/llvm/include/llvm/DebugInfo/GSYM/GsymContext.h b/llvm/include/llvm/DebugInfo/GSYM/GsymContext.h index 07d599c..e3e9b2b 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/GsymContext.h +++ b/llvm/include/llvm/DebugInfo/GSYM/GsymContext.h @@ -30,7 +30,7 @@ class GsymReader; class GsymContext : public DIContext { public: GsymContext(std::unique_ptr<GsymReader> Reader); - ~GsymContext(); + ~GsymContext() override; GsymContext(GsymContext &) = delete; GsymContext &operator=(GsymContext &) = delete; diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h index 0e7be45..34bace8 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h @@ -143,7 +143,7 @@ public: VirtualityCode(0) {} LVElement(const LVElement &) = delete; LVElement &operator=(const LVElement &) = delete; - virtual ~LVElement() = default; + ~LVElement() override = default; LVSubclassID getSubclassID() const { return SubclassID; } diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLine.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLine.h index 3618ce7..dd17f76 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLine.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLine.h @@ -53,7 +53,7 @@ public: } LVLine(const LVLine &) = delete; LVLine &operator=(const LVLine &) = delete; - virtual ~LVLine() = default; + ~LVLine() override = default; static bool classof(const LVElement *Element) { return Element->getSubclassID() == LVSubclassID::LV_LINE; @@ -117,7 +117,7 @@ public: LVLineDebug() : LVLine() { setIsLineDebug(); } LVLineDebug(const LVLineDebug &) = delete; LVLineDebug &operator=(const LVLineDebug &) = delete; - ~LVLineDebug() = default; + ~LVLineDebug() override = default; // Additional line information. It includes attributes that describes // states in the machine instructions (basic block, end prologue, etc). @@ -142,7 +142,7 @@ public: LVLineAssembler() : LVLine() { setIsLineAssembler(); } LVLineAssembler(const LVLineAssembler &) = delete; LVLineAssembler &operator=(const LVLineAssembler &) = delete; - ~LVLineAssembler() = default; + ~LVLineAssembler() override = default; // Print blanks as the line number. std::string noLineAsString(bool ShowZero) const override { diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h index 0718e33..090af54 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h @@ -100,7 +100,7 @@ public: LVLocation() : LVObject() { setIsLocation(); } LVLocation(const LVLocation &) = delete; LVLocation &operator=(const LVLocation &) = delete; - virtual ~LVLocation() = default; + ~LVLocation() override = default; PROPERTY(Property, IsAddressRange); PROPERTY(Property, IsBaseClassOffset); @@ -171,7 +171,7 @@ public: LVLocationSymbol() : LVLocation() {} LVLocationSymbol(const LVLocationSymbol &) = delete; LVLocationSymbol &operator=(const LVLocationSymbol &) = delete; - ~LVLocationSymbol() = default; + ~LVLocationSymbol() override = default; void addObject(LVAddress LowPC, LVAddress HighPC, LVUnsigned SectionOffset, uint64_t LocDescOffset) override; diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVRange.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVRange.h index b5c8333..4fa6a9a 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVRange.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVRange.h @@ -62,7 +62,7 @@ public: LVRange() : LVObject(), RangesTree(Allocator) {} LVRange(const LVRange &) = delete; LVRange &operator=(const LVRange &) = delete; - ~LVRange() = default; + ~LVRange() override = default; void addEntry(LVScope *Scope, LVAddress LowerAddress, LVAddress UpperAddress); void addEntry(LVScope *Scope); diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h index f4f3516..2e2619c 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h @@ -153,7 +153,7 @@ public: } LVScope(const LVScope &) = delete; LVScope &operator=(const LVScope &) = delete; - virtual ~LVScope() = default; + ~LVScope() override = default; static bool classof(const LVElement *Element) { return Element->getSubclassID() == LVSubclassID::LV_SCOPE; @@ -349,7 +349,7 @@ public: LVScopeAggregate() : LVScope() {} LVScopeAggregate(const LVScopeAggregate &) = delete; LVScopeAggregate &operator=(const LVScopeAggregate &) = delete; - ~LVScopeAggregate() = default; + ~LVScopeAggregate() override = default; // DW_AT_specification, DW_AT_abstract_origin. LVScope *getReference() const override { return Reference; } @@ -387,7 +387,7 @@ public: } LVScopeAlias(const LVScopeAlias &) = delete; LVScopeAlias &operator=(const LVScopeAlias &) = delete; - ~LVScopeAlias() = default; + ~LVScopeAlias() override = default; // Returns true if current scope is logically equal to the given 'Scope'. bool equals(const LVScope *Scope) const override; @@ -401,7 +401,7 @@ public: LVScopeArray() : LVScope() { setIsArray(); } LVScopeArray(const LVScopeArray &) = delete; LVScopeArray &operator=(const LVScopeArray &) = delete; - ~LVScopeArray() = default; + ~LVScopeArray() override = default; void resolveExtra() override; @@ -513,7 +513,7 @@ public: } LVScopeCompileUnit(const LVScopeCompileUnit &) = delete; LVScopeCompileUnit &operator=(const LVScopeCompileUnit &) = delete; - ~LVScopeCompileUnit() = default; + ~LVScopeCompileUnit() override = default; LVScope *getCompileUnitParent() const override { return static_cast<LVScope *>(const_cast<LVScopeCompileUnit *>(this)); @@ -643,7 +643,7 @@ public: LVScopeEnumeration() : LVScope() { setIsEnumeration(); } LVScopeEnumeration(const LVScopeEnumeration &) = delete; LVScopeEnumeration &operator=(const LVScopeEnumeration &) = delete; - ~LVScopeEnumeration() = default; + ~LVScopeEnumeration() override = default; // Returns true if current scope is logically equal to the given 'Scope'. bool equals(const LVScope *Scope) const override; @@ -658,7 +658,7 @@ public: LVScopeFormalPack() : LVScope() { setIsTemplatePack(); } LVScopeFormalPack(const LVScopeFormalPack &) = delete; LVScopeFormalPack &operator=(const LVScopeFormalPack &) = delete; - ~LVScopeFormalPack() = default; + ~LVScopeFormalPack() override = default; // Returns true if current scope is logically equal to the given 'Scope'. bool equals(const LVScope *Scope) const override; @@ -676,7 +676,7 @@ public: LVScopeFunction() : LVScope() {} LVScopeFunction(const LVScopeFunction &) = delete; LVScopeFunction &operator=(const LVScopeFunction &) = delete; - virtual ~LVScopeFunction() = default; + ~LVScopeFunction() override = default; // DW_AT_specification, DW_AT_abstract_origin. LVScope *getReference() const override { return Reference; } @@ -728,7 +728,7 @@ public: LVScopeFunctionInlined() : LVScopeFunction() { setIsInlinedFunction(); } LVScopeFunctionInlined(const LVScopeFunctionInlined &) = delete; LVScopeFunctionInlined &operator=(const LVScopeFunctionInlined &) = delete; - ~LVScopeFunctionInlined() = default; + ~LVScopeFunctionInlined() override = default; uint32_t getDiscriminator() const override { return Discriminator; } void setDiscriminator(uint32_t Value) override { @@ -767,7 +767,7 @@ public: LVScopeFunctionType() : LVScopeFunction() { setIsFunctionType(); } LVScopeFunctionType(const LVScopeFunctionType &) = delete; LVScopeFunctionType &operator=(const LVScopeFunctionType &) = delete; - ~LVScopeFunctionType() = default; + ~LVScopeFunctionType() override = default; void resolveExtra() override; }; @@ -781,7 +781,7 @@ public: } LVScopeModule(const LVScopeModule &) = delete; LVScopeModule &operator=(const LVScopeModule &) = delete; - ~LVScopeModule() = default; + ~LVScopeModule() override = default; // Returns true if current scope is logically equal to the given 'Scope'. bool equals(const LVScope *Scope) const override; @@ -797,7 +797,7 @@ public: LVScopeNamespace() : LVScope() { setIsNamespace(); } LVScopeNamespace(const LVScopeNamespace &) = delete; LVScopeNamespace &operator=(const LVScopeNamespace &) = delete; - ~LVScopeNamespace() = default; + ~LVScopeNamespace() override = default; // Access DW_AT_extension reference. LVScope *getReference() const override { return Reference; } @@ -827,7 +827,7 @@ public: LVScopeRoot() : LVScope() { setIsRoot(); } LVScopeRoot(const LVScopeRoot &) = delete; LVScopeRoot &operator=(const LVScopeRoot &) = delete; - ~LVScopeRoot() = default; + ~LVScopeRoot() override = default; StringRef getFileFormatName() const { return getStringPool().getString(FileFormatNameIndex); @@ -859,7 +859,7 @@ public: LVScopeTemplatePack() : LVScope() { setIsTemplatePack(); } LVScopeTemplatePack(const LVScopeTemplatePack &) = delete; LVScopeTemplatePack &operator=(const LVScopeTemplatePack &) = delete; - ~LVScopeTemplatePack() = default; + ~LVScopeTemplatePack() override = default; // Returns true if current scope is logically equal to the given 'Scope'. bool equals(const LVScope *Scope) const override; diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h index ec9017e..c5314fc 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h @@ -74,7 +74,7 @@ public: } LVSymbol(const LVSymbol &) = delete; LVSymbol &operator=(const LVSymbol &) = delete; - ~LVSymbol() = default; + ~LVSymbol() override = default; static bool classof(const LVElement *Element) { return Element->getSubclassID() == LVSubclassID::LV_SYMBOL; diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVType.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVType.h index 59e6a92..af4abcf 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVType.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVType.h @@ -67,7 +67,7 @@ public: LVType() : LVElement(LVSubclassID::LV_TYPE) { setIsType(); } LVType(const LVType &) = delete; LVType &operator=(const LVType &) = delete; - virtual ~LVType() = default; + ~LVType() override = default; static bool classof(const LVElement *Element) { return Element->getSubclassID() == LVSubclassID::LV_TYPE; @@ -157,7 +157,7 @@ public: } LVTypeDefinition(const LVTypeDefinition &) = delete; LVTypeDefinition &operator=(const LVTypeDefinition &) = delete; - ~LVTypeDefinition() = default; + ~LVTypeDefinition() override = default; // Return the underlying type for a type definition. LVElement *getUnderlyingType() override; @@ -183,7 +183,7 @@ public: } LVTypeEnumerator(const LVTypeEnumerator &) = delete; LVTypeEnumerator &operator=(const LVTypeEnumerator &) = delete; - ~LVTypeEnumerator() = default; + ~LVTypeEnumerator() override = default; // Process the values for a DW_TAG_enumerator. StringRef getValue() const override { @@ -206,7 +206,7 @@ public: LVTypeImport() : LVType() { setIncludeInPrint(); } LVTypeImport(const LVTypeImport &) = delete; LVTypeImport &operator=(const LVTypeImport &) = delete; - ~LVTypeImport() = default; + ~LVTypeImport() override = default; // Returns true if current type is logically equal to the given 'Type'. bool equals(const LVType *Type) const override; @@ -223,7 +223,7 @@ public: LVTypeParam(); LVTypeParam(const LVTypeParam &) = delete; LVTypeParam &operator=(const LVTypeParam &) = delete; - ~LVTypeParam() = default; + ~LVTypeParam() override = default; // Template parameter value. StringRef getValue() const override { @@ -256,7 +256,7 @@ public: } LVTypeSubrange(const LVTypeSubrange &) = delete; LVTypeSubrange &operator=(const LVTypeSubrange &) = delete; - ~LVTypeSubrange() = default; + ~LVTypeSubrange() override = default; int64_t getCount() const override { return getIsSubrangeCount() ? LowerBound : 0; diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h index 2cf4a8e..cc8dda2 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h @@ -192,7 +192,7 @@ public: : LVReader(Filename, FileFormatName, W, BinaryType) {} LVBinaryReader(const LVBinaryReader &) = delete; LVBinaryReader &operator=(const LVBinaryReader &) = delete; - virtual ~LVBinaryReader() = default; + ~LVBinaryReader() override = default; void addInlineeLines(LVScope *Scope, LVLines &Lines) { CUInlineeLines.emplace(Scope, std::make_unique<LVLines>(std::move(Lines))); diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h index 4dd7c96..9f6fd553 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h @@ -200,7 +200,7 @@ public: Input(&Pdb), ExePath(ExePath), LogicalVisitor(this, W, Input) {} LVCodeViewReader(const LVCodeViewReader &) = delete; LVCodeViewReader &operator=(const LVCodeViewReader &) = delete; - ~LVCodeViewReader() = default; + ~LVCodeViewReader() override = default; void getLinkageName(const llvm::object::coff_section *CoffSection, uint32_t RelocOffset, uint32_t Offset, diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h index 2abc18b..1cf2914 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h @@ -123,7 +123,7 @@ public: Obj(Obj) {} LVDWARFReader(const LVDWARFReader &) = delete; LVDWARFReader &operator=(const LVDWARFReader &) = delete; - ~LVDWARFReader() = default; + ~LVDWARFReader() override = default; LVAddress getCUBaseAddress() const { return CUBaseAddress; } void setCUBaseAddress(LVAddress Address) { CUBaseAddress = Address; } diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h index b21cd09..c0b3151 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h @@ -20,7 +20,7 @@ namespace pdb { class LLVM_ABI PDBSymbolTypeBuiltin : public PDBSymbol { DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::BuiltinType) public: - ~PDBSymbolTypeBuiltin(); + ~PDBSymbolTypeBuiltin() override; void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getBuiltinType) diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h index 8613ddd..24a0cb7 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h @@ -448,7 +448,7 @@ public: FailedToMaterialize(std::shared_ptr<SymbolStringPool> SSP, std::shared_ptr<SymbolDependenceMap> Symbols); - ~FailedToMaterialize(); + ~FailedToMaterialize() override; std::error_code convertToErrorCode() const override; void log(raw_ostream &OS) const override; const SymbolDependenceMap &getSymbols() const { return *Symbols; } @@ -1768,7 +1768,7 @@ private: // FIXME: We should be able to derive FailedSymsForQuery from each query once // we fix how the detach operation works. struct EmitQueries { - JITDylib::AsynchronousSymbolQuerySet Updated; + JITDylib::AsynchronousSymbolQuerySet Completed; JITDylib::AsynchronousSymbolQuerySet Failed; DenseMap<AsynchronousSymbolQuery *, std::shared_ptr<SymbolDependenceMap>> FailedSymsForQuery; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h b/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h index 254b897..e84eb4b 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h @@ -70,7 +70,7 @@ public: DebugObjectManagerPlugin(ExecutionSession &ES, std::unique_ptr<DebugObjectRegistrar> Target, bool RequireDebugSections, bool AutoRegisterCode); - ~DebugObjectManagerPlugin(); + ~DebugObjectManagerPlugin() override; void notifyMaterializing(MaterializationResponsibility &MR, jitlink::LinkGraph &G, jitlink::JITLinkContext &Ctx, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h b/llvm/include/llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h index 179fedc..f9b1d2c 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h @@ -32,7 +32,7 @@ public: ExecutorAddr RegisterPerfEndAddr, ExecutorAddr RegisterPerfImplAddr, bool EmitDebugInfo, bool EmitUnwindInfo); - ~PerfSupportPlugin(); + ~PerfSupportPlugin() override; void modifyPassConfig(MaterializationResponsibility &MR, jitlink::LinkGraph &G, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h index fa48480..031bb27 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h @@ -52,7 +52,7 @@ public: EPCGenericRTDyldMemoryManager(EPCGenericRTDyldMemoryManager &&) = delete; EPCGenericRTDyldMemoryManager & operator=(EPCGenericRTDyldMemoryManager &&) = delete; - ~EPCGenericRTDyldMemoryManager(); + ~EPCGenericRTDyldMemoryManager() override; uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, unsigned SectionID, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index fecffc2..fa4bb9d 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -285,7 +285,7 @@ public: /// Map type for initializing the manager. See init. using StubInitsMap = StringMap<std::pair<ExecutorAddr, JITSymbolFlags>>; - virtual ~IndirectStubsManager() = default; + ~IndirectStubsManager() override = default; /// Create a single stub with the given name, target address and flags. virtual Error createStub(StringRef StubName, ExecutorAddr StubAddr, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Layer.h b/llvm/include/llvm/ExecutionEngine/Orc/Layer.h index 8dfc12e..25df380 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Layer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Layer.h @@ -136,7 +136,7 @@ public: static char ID; ObjectLayer(ExecutionSession &ES); - virtual ~ObjectLayer(); + ~ObjectLayer() override; /// Returns the execution session for this layer. ExecutionSession &getExecutionSession() { return ES; } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h index d3643f9..2079a35 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h @@ -88,7 +88,7 @@ public: std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr); /// Destroy the LinkGraphLinkingLayer. - ~LinkGraphLinkingLayer(); + ~LinkGraphLinkingLayer() override; /// Add a plugin. LinkGraphLinkingLayer &addPlugin(std::shared_ptr<Plugin> P) { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h index 1fb472a1..8c6a8f5 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h @@ -58,7 +58,7 @@ public: RTDyldObjectLinkingLayer(ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager); - ~RTDyldObjectLinkingLayer(); + ~RTDyldObjectLinkingLayer() override; /// Emit the object. void emit(std::unique_ptr<MaterializationResponsibility> R, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h index 4a32113b..6adaa8a 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h @@ -34,7 +34,7 @@ using ExecutorAddrDiff = uint64_t; class ExecutorAddr { public: /// A wrap/unwrap function that leaves pointers unmodified. - using rawPtr = llvm::identity_cxx20; + using rawPtr = llvm::identity; #if __has_feature(ptrauth_calls) template <typename T> class PtrauthSignDefault { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h b/llvm/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h index 7acb6a4..8176183 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h @@ -69,7 +69,7 @@ public: SimpleRemoteEPC &operator=(const SimpleRemoteEPC &) = delete; SimpleRemoteEPC(SimpleRemoteEPC &&) = delete; SimpleRemoteEPC &operator=(SimpleRemoteEPC &&) = delete; - ~SimpleRemoteEPC(); + ~SimpleRemoteEPC() override; Expected<int32_t> runAsMain(ExecutorAddr MainFnAddr, ArrayRef<std::string> Args) override; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h index 85c2d65..2c385de 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h @@ -29,7 +29,7 @@ namespace rt_bootstrap { class LLVM_ABI ExecutorSharedMemoryMapperService final : public ExecutorBootstrapService { public: - ~ExecutorSharedMemoryMapperService(){}; + ~ExecutorSharedMemoryMapperService() override {}; Expected<std::pair<ExecutorAddr, std::string>> reserve(uint64_t Size); Expected<ExecutorAddr> initialize(ExecutorAddr Reservation, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h index 7526a29d..7ca2ff2 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h @@ -37,7 +37,7 @@ namespace rt_bootstrap { /// Simple page-based allocator. class LLVM_ABI SimpleExecutorDylibManager : public ExecutorBootstrapService { public: - virtual ~SimpleExecutorDylibManager(); + ~SimpleExecutorDylibManager() override; Expected<tpctypes::DylibHandle> open(const std::string &Path, uint64_t Mode); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h index 6224e92..45256ec 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h @@ -32,7 +32,7 @@ namespace rt_bootstrap { /// Simple page-based allocator. class LLVM_ABI SimpleExecutorMemoryManager : public ExecutorBootstrapService { public: - virtual ~SimpleExecutorMemoryManager(); + ~SimpleExecutorMemoryManager() override; Expected<ExecutorAddr> reserve(uint64_t Size); Expected<ExecutorAddr> initialize(tpctypes::FinalizeRequest &FR); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TaskDispatch.h b/llvm/include/llvm/ExecutionEngine/Orc/TaskDispatch.h index 9cf6e00..b73da19 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TaskDispatch.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TaskDispatch.h @@ -37,7 +37,7 @@ class LLVM_ABI Task : public RTTIExtends<Task, RTTIRoot> { public: static char ID; - virtual ~Task() = default; + ~Task() override = default; /// Description of the task to be performed. Used for logging. virtual void printDescription(raw_ostream &OS) = 0; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h index 45834f1..d8506ce 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h @@ -186,9 +186,7 @@ private: SmallVector<ElementId> SortedElems(ContainerElems.begin(), ContainerElems.end()); llvm::sort(SortedElems); - Hash = hash_combine( - Hash, Container, - hash_combine_range(SortedElems.begin(), SortedElems.end())); + Hash = hash_combine(Hash, Container, hash_combine_range(SortedElems)); } return Hash; } diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index e6cce9a4..4d59ee8 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1487,24 +1487,23 @@ def int_eh_sjlj_setup_dispatch : Intrinsic<[], []>; // def int_var_annotation : DefaultAttrsIntrinsic< [], [llvm_anyptr_ty, llvm_anyptr_ty, LLVMMatchType<1>, llvm_i32_ty, LLVMMatchType<1>], - [IntrInaccessibleMemOnly], "llvm.var.annotation">; + [IntrInaccessibleMemOnly]>; def int_ptr_annotation : DefaultAttrsIntrinsic< [llvm_anyptr_ty], [LLVMMatchType<0>, llvm_anyptr_ty, LLVMMatchType<1>, llvm_i32_ty, LLVMMatchType<1>], - [IntrInaccessibleMemOnly], "llvm.ptr.annotation">; + [IntrInaccessibleMemOnly]>; def int_annotation : DefaultAttrsIntrinsic< [llvm_anyint_ty], [LLVMMatchType<0>, llvm_anyptr_ty, LLVMMatchType<1>, llvm_i32_ty], - [IntrInaccessibleMemOnly], "llvm.annotation">; + [IntrInaccessibleMemOnly]>; // Annotates the current program point with metadata strings which are emitted // as CodeView debug info records. This is expensive, as it disables inlining // and is modelled as having side effects. def int_codeview_annotation : DefaultAttrsIntrinsic<[], [llvm_metadata_ty], - [IntrInaccessibleMemOnly, IntrNoDuplicate], - "llvm.codeview.annotation">; + [IntrInaccessibleMemOnly, IntrNoDuplicate]>; //===------------------------ Trampoline Intrinsics -----------------------===// // @@ -1881,8 +1880,7 @@ def int_clear_cache : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], // Intrinsic to detect whether its argument is a constant. def int_is_constant : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty], - [IntrNoMem, IntrConvergent], - "llvm.is.constant">; + [IntrNoMem, IntrConvergent]>; // Introduce a use of the argument without generating any code. def int_fake_use : DefaultAttrsIntrinsic<[], [llvm_vararg_ty], diff --git a/llvm/include/llvm/IR/IntrinsicsHexagon.td b/llvm/include/llvm/IR/IntrinsicsHexagon.td index 67b873d..20ba51a 100644 --- a/llvm/include/llvm/IR/IntrinsicsHexagon.td +++ b/llvm/include/llvm/IR/IntrinsicsHexagon.td @@ -447,3 +447,15 @@ def int_hexagon_instrprof_custom include "llvm/IR/IntrinsicsHexagonDep.td" + +class Hexagon__ptri32i32v64i16_Intrinsic<string GCCIntSuffix, + list<IntrinsicProperty> intr_properties = [IntrNoMem]> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_ptr_ty,llvm_i32_ty,llvm_i32_ty,llvm_v64i16_ty], + intr_properties>; + +def int_hexagon_V6_vgather_vscattermh : +Hexagon__ptri32i32v64i16_Intrinsic<"HEXAGON_V6_vgather_vscattermh", [IntrArgMemOnly]>; + +def int_hexagon_V6_vgather_vscattermh_128B : +Hexagon__ptri32i32v32i32_Intrinsic<"HEXAGON_V6_vgather_vscattermh_128B", [IntrArgMemOnly]>; diff --git a/llvm/include/llvm/IR/IntrinsicsHexagonDep.td b/llvm/include/llvm/IR/IntrinsicsHexagonDep.td index 0cd6008..fe95377 100644 --- a/llvm/include/llvm/IR/IntrinsicsHexagonDep.td +++ b/llvm/include/llvm/IR/IntrinsicsHexagonDep.td @@ -6832,3 +6832,17 @@ Hexagon_v32i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_hf_f8">; def int_hexagon_V6_vsub_hf_f8_128B : Hexagon_v64i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_hf_f8_128B">; + +// V81 HVX Instructions. + +def int_hexagon_V6_vsub_hf_mix : +Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_hf_mix">; + +def int_hexagon_V6_vsub_hf_mix_128B : +Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_hf_mix_128B">; + +def int_hexagon_V6_vsub_sf_mix : +Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_sf_mix">; + +def int_hexagon_V6_vsub_sf_mix_128B : +Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_sf_mix_128B">; diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td index 3af1750..c9df6c4 100644 --- a/llvm/include/llvm/IR/IntrinsicsNVVM.td +++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td @@ -456,7 +456,7 @@ class WMMA_REGS<string Geom, string Frag, string PtxEltType, bit IsSparse = fals } class WMMA_NAME_LDST<string Op, WMMA_REGS Frag, string Layout, int WithStride> { - string intr = "llvm.nvvm.wmma." + string intr_name = "llvm.nvvm.wmma." # Frag.geom # "." # Op # "." # Frag.frag @@ -467,7 +467,7 @@ class WMMA_NAME_LDST<string Op, WMMA_REGS Frag, string Layout, int WithStride> { // TODO(tra): record name should ideally use the same field order as the intrinsic. // E.g. string record = !subst("llvm", "int", // !subst(".", "_", llvm)); - string record = "int_nvvm_wmma_" + string record_name = "int_nvvm_wmma_" # Frag.geom # "_" # Op # "_" # Frag.frag @@ -496,7 +496,7 @@ class MMA_SIGNATURE<WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> { class WMMA_NAME<string ALayout, string BLayout, int Satfinite, string Rnd, string b1op, WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> { string signature = MMA_SIGNATURE<A, B, C, D>.ret; - string record = "int_nvvm_wmma_" + string record_name = "int_nvvm_wmma_" # A.geom # "_mma" # !subst(".", "_", b1op) @@ -510,7 +510,7 @@ class WMMA_NAME<string ALayout, string BLayout, int Satfinite, string Rnd, strin class MMA_NAME<string ALayout, string BLayout, int Satfinite, string b1op, string Kind, WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> { string signature = MMA_SIGNATURE<A, B, C, D>.ret; - string record = "int_nvvm_mma" + string record_name = "int_nvvm_mma" # !subst(".", "_", b1op) # "_" # A.geom # "_" # ALayout @@ -524,7 +524,7 @@ class MMA_SP_NAME<string Metadata, string Kind, int Satfinite, WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> { string signature = MMA_SIGNATURE<A, B, C, D>.ret; - string record = "int_nvvm_mma" + string record_name = "int_nvvm_mma" # "_" # !subst("::", "_", Metadata) # "_" # A.geom # "_row_col" @@ -533,26 +533,37 @@ class MMA_SP_NAME<string Metadata, string Kind, int Satfinite, # signature; } +// Helper class that takes an intrinsic name and construct a record name. +// Additionally, sets `intr_name` to be non-empty if the default name assigned +// to this intrinsic will not match the name given. +class IntrinsicName<string name> { + string record_name = !subst(".", "_", + !subst("llvm.", "int_", name)); + // Use explicit intrinsic name if it has an _ in it, else rely on LLVM + // assigned default name. + string intr_name = !if(!ne(!find(name, "_"), -1), name, ""); +} + class LDMATRIX_NAME<WMMA_REGS Frag, int Trans> { - string intr = "llvm.nvvm.ldmatrix.sync.aligned" + defvar name = "llvm.nvvm.ldmatrix.sync.aligned" # "." # Frag.geom # "." # Frag.frag # !if(Trans, ".trans", "") # "." # Frag.ptx_elt_type ; - string record = !subst(".", "_", - !subst("llvm.", "int_", intr)); + string intr_name = IntrinsicName<name>.intr_name; + string record_name = IntrinsicName<name>.record_name; } class STMATRIX_NAME<WMMA_REGS Frag, int Trans> { - string intr = "llvm.nvvm.stmatrix.sync.aligned" + defvar name = "llvm.nvvm.stmatrix.sync.aligned" # "." # Frag.geom # "." # Frag.frag # !if(Trans, ".trans", "") # "." # Frag.ptx_elt_type ; - string record = !subst(".", "_", - !subst("llvm.", "int_", intr)); + string intr_name = IntrinsicName<name>.intr_name; + string record_name = IntrinsicName<name>.record_name; } // Generates list of 4-tuples of WMMA_REGS representing a valid MMA op. @@ -1042,45 +1053,49 @@ class NVVM_TCGEN05_MMA_BASE<string Space, bit Sp> { class NVVM_TCGEN05_MMA<bit Sp, string Space, bit AShift, bit ScaleInputD>: NVVM_TCGEN05_MMA_BASE<Space, Sp> { - string intr = "llvm.nvvm.tcgen05.mma" + string name = "llvm.nvvm.tcgen05.mma" # !if(!eq(Sp, 1), ".sp", "") # "." # Space # !if(!eq(ScaleInputD, 1), ".scale_d", "") # !if(!eq(AShift, 1), ".ashift", ""); - string record = !subst(".", "_", !subst("llvm.", "int_", intr)); + string intr_name = IntrinsicName<name>.intr_name; + string record_name = IntrinsicName<name>.record_name; } class NVVM_TCGEN05_MMA_BLOCKSCALE<bit Sp, string Space, string Kind, string ScaleVecSize>: NVVM_TCGEN05_MMA_BASE<Space, Sp> { - string intr = "llvm.nvvm.tcgen05.mma" + string name = "llvm.nvvm.tcgen05.mma" # !if(!eq(Sp, 1), ".sp", "") # "." # Space # "." # Kind # ".block_scale" # ScaleVecSize; - string record = !subst(".", "_", !subst("llvm.", "int_", intr)); + string intr_name = IntrinsicName<name>.intr_name; + string record_name = IntrinsicName<name>.record_name; } class NVVM_TCGEN05_MMA_WS<bit Sp, string Space, bit ZeroColMask>: NVVM_TCGEN05_MMA_BASE<Space, Sp> { - string intr = "llvm.nvvm.tcgen05.mma.ws" + string name = "llvm.nvvm.tcgen05.mma.ws" # !if(!eq(Sp, 1), ".sp", "") # "." # Space # !if(!eq(ZeroColMask, 1), ".zero_col_mask", ""); - string record = !subst(".", "_", !subst("llvm.", "int_", intr)); + string intr_name = IntrinsicName<name>.intr_name; + string record_name = IntrinsicName<name>.record_name; } class NVVM_TCGEN05_MMA_DISABLE_OUTPUT_LANE<bit Sp, string Space, int CtaGroup, bit AShift, bit ScaleInputD>: NVVM_TCGEN05_MMA_BASE<Space, Sp> { - string intr = "llvm.nvvm.tcgen05.mma" + string name = "llvm.nvvm.tcgen05.mma" # !if(!eq(Sp, 1), ".sp", "") # "." # Space # !if(!eq(ScaleInputD, 1), ".scale_d", "") # ".disable_output_lane.cg" # CtaGroup # !if(!eq(AShift, 1), ".ashift", ""); - string record = !subst(".", "_", !subst("llvm.", "int_", intr)); + string intr_name = IntrinsicName<name>.intr_name; + string record_name = IntrinsicName<name>.record_name; } class NVVM_TCGEN05_MMA_BLOCKSCALE_SUPPORTED<string Kind, string ScaleVecSize> { @@ -2273,7 +2288,7 @@ class NVVM_WMMA_LD<WMMA_REGS Frag, string Layout, int WithStride> : Intrinsic<Frag.regs, !if(WithStride, [llvm_anyptr_ty, llvm_i32_ty], [llvm_anyptr_ty]), [IntrWillReturn, IntrReadMem, IntrArgMemOnly, IntrNoCallback, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], - WMMA_NAME_LDST<"load", Frag, Layout, WithStride>.intr>; + WMMA_NAME_LDST<"load", Frag, Layout, WithStride>.intr_name>; // WMMA.STORE.D class NVVM_WMMA_ST<WMMA_REGS Frag, string Layout, int WithStride> @@ -2283,18 +2298,18 @@ class NVVM_WMMA_ST<WMMA_REGS Frag, string Layout, int WithStride> Frag.regs, !if(WithStride, [llvm_i32_ty], [])), [IntrWriteMem, IntrArgMemOnly, IntrNoCallback, WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], - WMMA_NAME_LDST<"store", Frag, Layout, WithStride>.intr>; + WMMA_NAME_LDST<"store", Frag, Layout, WithStride>.intr_name>; // Create all load/store variants foreach layout = ["row", "col"] in { foreach stride = [0, 1] in { foreach frag = NVVM_MMA_OPS.all_ld_ops in if NVVM_WMMA_LDST_SUPPORTED<frag, layout>.ret then - def WMMA_NAME_LDST<"load", frag, layout, stride>.record + def WMMA_NAME_LDST<"load", frag, layout, stride>.record_name : NVVM_WMMA_LD<frag, layout, stride>; foreach frag = NVVM_MMA_OPS.all_st_ops in if NVVM_WMMA_LDST_SUPPORTED<frag, layout>.ret then - def WMMA_NAME_LDST<"store", frag, layout, stride>.record + def WMMA_NAME_LDST<"store", frag, layout, stride>.record_name : NVVM_WMMA_ST<frag, layout, stride>; } } @@ -2313,7 +2328,7 @@ foreach layout_a = ["row", "col"] in { foreach b1op = NVVM_MMA_B1OPS<op>.ret in { if NVVM_WMMA_SUPPORTED<op, layout_a, layout_b, satf, rnd>.ret then { def WMMA_NAME<layout_a, layout_b, satf, rnd, b1op, - op[0], op[1], op[2], op[3]>.record + op[0], op[1], op[2], op[3]>.record_name : NVVM_MMA<op[0], op[1], op[2], op[3]>; } } // b1op @@ -2330,7 +2345,7 @@ foreach layout_a = ["row", "col"] in { foreach b1op = NVVM_MMA_B1OPS<op>.ret in { foreach kind = ["", "kind::f8f6f4"] in { if NVVM_MMA_SUPPORTED<op, layout_a, layout_b, kind, satf>.ret then { - def MMA_NAME<layout_a, layout_b, satf, b1op, kind, op[0], op[1], op[2], op[3]>.record + def MMA_NAME<layout_a, layout_b, satf, b1op, kind, op[0], op[1], op[2], op[3]>.record_name : NVVM_MMA<op[0], op[1], op[2], op[3]>; } } // kind @@ -2379,7 +2394,7 @@ foreach metadata = ["sp", "sp::ordered_metadata"] in { foreach op = NVVM_MMA_OPS.all_mma_sp_ops in { if NVVM_MMA_SP_SUPPORTED<op, metadata, kind, satf>.ret then { def MMA_SP_NAME<metadata, kind, satf, - op[0], op[1], op[2], op[3]>.record + op[0], op[1], op[2], op[3]>.record_name : NVVM_MMA_SP<op[0], op[1], op[2], op[3]>; } } // op @@ -2392,12 +2407,12 @@ class NVVM_LDMATRIX<WMMA_REGS Frag, int Transposed> : Intrinsic<Frag.regs, [llvm_anyptr_ty], [IntrReadMem, IntrArgMemOnly, IntrNoCallback, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], - LDMATRIX_NAME<Frag, Transposed>.intr>; + LDMATRIX_NAME<Frag, Transposed>.intr_name>; foreach transposed = [0, 1] in { foreach frag = NVVM_MMA_OPS.all_ldmatrix_ops in { if NVVM_LDMATRIX_SUPPORTED<frag, transposed>.ret then { - def LDMATRIX_NAME<frag, transposed>.record + def LDMATRIX_NAME<frag, transposed>.record_name : NVVM_LDMATRIX<frag, transposed>; } } @@ -2409,12 +2424,12 @@ class NVVM_STMATRIX<WMMA_REGS Frag, int Transposed> !listconcat([llvm_anyptr_ty], Frag.regs), [IntrWriteMem, IntrArgMemOnly, IntrNoCallback, WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], - STMATRIX_NAME<Frag, Transposed>.intr>; + STMATRIX_NAME<Frag, Transposed>.intr_name>; foreach transposed = [0, 1] in { foreach frag = NVVM_MMA_OPS.all_stmatrix_ops in { if NVVM_STMATRIX_SUPPORTED<frag, transposed>.ret then { - def STMATRIX_NAME<frag, transposed>.record + def STMATRIX_NAME<frag, transposed>.record_name : NVVM_STMATRIX<frag, transposed>; } } @@ -2767,14 +2782,15 @@ foreach cta_group = ["cg1", "cg2"] in { "64x128b_warpx2_02_13", "64x128b_warpx2_01_23", "32x128b_warpx4"] in { - defvar intr_suffix = StrJoin<"_", [shape, src_fmt, cta_group]>.ret; - defvar name_suffix = StrJoin<".", [shape, src_fmt, cta_group]>.ret; + defvar name = "llvm.nvvm.tcgen05.cp." # + StrJoin<".", [shape, src_fmt, cta_group]>.ret; - def int_nvvm_tcgen05_cp_ # intr_suffix : Intrinsic<[], + defvar intrinsic_name = IntrinsicName<name>; + def intrinsic_name.record_name : Intrinsic<[], [llvm_tmem_ptr_ty, // tmem_addr llvm_i64_ty], // smem descriptor [IntrConvergent, IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<0>>], - "llvm.nvvm.tcgen05.cp." # name_suffix>; + intrinsic_name.intr_name>; } } } @@ -2881,9 +2897,9 @@ foreach sp = [0, 1] in { ] ); - def mma.record: + def mma.record_name: DefaultAttrsIntrinsicFlags<[], args, flags, intrinsic_properties, - mma.intr>; + mma.intr_name>; } } } @@ -2918,8 +2934,8 @@ foreach sp = [0, 1] in { Range<ArgIndex<!add(nargs, 1)>, 0, !if(!eq(ashift, 1), 2, 4)>] ); - def mma.record: DefaultAttrsIntrinsicFlags<[], args, flags, intrinsic_properties, - mma.intr>; + def mma.record_name : DefaultAttrsIntrinsicFlags<[], args, flags, + intrinsic_properties, mma.intr_name>; } // ashift } // scale_d } // cta_group @@ -2944,11 +2960,11 @@ foreach sp = [0, 1] in { defvar collector_usage = ArgIndex<!add(nargs, 1)>; if NVVM_TCGEN05_MMA_BLOCKSCALE_SUPPORTED<kind, scale_vec_size>.ret then { - def mma.record: DefaultAttrsIntrinsicFlags<[], args, flags, + def mma.record_name : DefaultAttrsIntrinsicFlags<[], args, flags, !listconcat(mma.common_intr_props, [Range<cta_group, 1, 3>, Range<collector_usage, 0, 4>]), - mma.intr>; + mma.intr_name>; } } } @@ -2977,9 +2993,9 @@ foreach sp = [0, 1] in { Range<ArgIndex<!add(nargs, 2)>, 0, 4>] ); - def mma.record: + def mma.record_name: DefaultAttrsIntrinsicFlags<[], args, flags, intrinsic_properties, - mma.intr>; + mma.intr_name>; } } } diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td b/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td index 4a0272c..b1bb24e 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td @@ -162,6 +162,10 @@ let TargetPrefix = "riscv" in { defm "" : RISCVSFCustomVC_XVV<["x", "i", "v", "f"]>; defm "" : RISCVSFCustomVC_XVW<["x", "i", "v", "f"]>; + // XSfvfexp* and XSfvfexpa* + defm sf_vfexp : RISCVUnaryAA; + defm sf_vfexpa : RISCVUnaryAA; + // XSfvqmaccdod def int_riscv_sf_vqmaccu_2x8x2 : RISCVSFCustomVMACC; def int_riscv_sf_vqmacc_2x8x2 : RISCVSFCustomVMACC; diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h index cdfee72..eb60bee 100644 --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -178,6 +178,8 @@ struct alignas(8) GlobalValueSummaryInfo { /// only be called prior to index-based internalization and promotion. inline void verifyLocal() const; + bool hasLocal() const { return HasLocal; } + private: /// List of global value summary structures for a particular value held /// in the GlobalValueMap. Requires a vector in the case of multiple @@ -239,6 +241,8 @@ struct ValueInfo { void verifyLocal() const { getRef()->second.verifyLocal(); } + bool hasLocal() const { return getRef()->second.hasLocal(); } + // Even if the index is built with GVs available, we may not have one for // summary entries synthesized for profiled indirect call targets. bool hasName() const { return !haveGVs() || getValue(); } diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 99f70b1..e3ec7e1 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -198,6 +198,26 @@ struct constantexpr_match { /// expression. inline constantexpr_match m_ConstantExpr() { return constantexpr_match(); } +template <typename SubPattern_t> struct Splat_match { + SubPattern_t SubPattern; + Splat_match(const SubPattern_t &SP) : SubPattern(SP) {} + + template <typename OpTy> bool match(OpTy *V) const { + if (auto *C = dyn_cast<Constant>(V)) { + auto *Splat = C->getSplatValue(); + return Splat ? SubPattern.match(Splat) : false; + } + // TODO: Extend to other cases (e.g. shufflevectors). + return false; + } +}; + +/// Match a constant splat. TODO: Extend this to non-constant splats. +template <typename T> +inline Splat_match<T> m_ConstantSplat(const T &SubPattern) { + return SubPattern; +} + /// Match an arbitrary basic block value and ignore it. inline class_match<BasicBlock> m_BasicBlock() { return class_match<BasicBlock>(); @@ -2925,6 +2945,12 @@ inline typename m_Intrinsic_Ty<Opnd0>::Ty m_VecReverse(const Opnd0 &Op0) { return m_Intrinsic<Intrinsic::vector_reverse>(Op0); } +template <typename Opnd0, typename Opnd1, typename Opnd2> +inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty +m_VectorInsert(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { + return m_Intrinsic<Intrinsic::vector_insert>(Op0, Op1, Op2); +} + //===----------------------------------------------------------------------===// // Matchers for two-operands operators with the operators in either order // diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.td b/llvm/include/llvm/IR/RuntimeLibcalls.td index 3dc9055..ec16995 100644 --- a/llvm/include/llvm/IR/RuntimeLibcalls.td +++ b/llvm/include/llvm/IR/RuntimeLibcalls.td @@ -32,6 +32,15 @@ def isWindowsMSVCEnvironment : RuntimeLibcallPredicate< def isNotOSLinuxAndNotOSOpenBSD : RuntimeLibcallPredicate< [{!TT.isOSLinux() && !TT.isOSOpenBSD()}]>; +def isNotOSAIXAndNotOSOpenBSD : RuntimeLibcallPredicate< + [{!TT.isOSAIX() && !TT.isOSOpenBSD()}]>; + +// OpenBSD uses __guard_local. AIX uses __ssp_canary_word, MSVC/Windows +// Itanium uses __security_cookie +def hasStackChkFail : RuntimeLibcallPredicate< + [{ !TT.isOSOpenBSD() && !TT.isWindowsMSVCEnvironment() && + !TT.isWindowsItaniumEnvironment()}]>; + def isWindowsMSVCOrItaniumEnvironment : RuntimeLibcallPredicate< [{TT.isWindowsMSVCEnvironment() || TT.isWindowsItaniumEnvironment()}]>; @@ -1241,9 +1250,9 @@ defvar LibmHasLdexpF80 = LibcallImpls<(add ldexpl_f80), isNotOSWindowsOrIsCygwin defvar LibmHasFrexpF128 = LibcallImpls<(add frexpl_f128), isNotOSWindowsOrIsCygwinMinGW>; defvar LibmHasLdexpF128 = LibcallImpls<(add ldexpl_f128), isNotOSWindowsOrIsCygwinMinGW>; -defvar has__stack_chk_fail = LibcallImpls<(add __stack_chk_fail), isNotOSOpenBSD>; +defvar has__stack_chk_fail = LibcallImpls<(add __stack_chk_fail), hasStackChkFail>; defvar has__stack_chk_guard = - LibcallImpls<(add __stack_chk_guard), isNotOSOpenBSD>; + LibcallImpls<(add __stack_chk_guard), hasStackChkFail>; defvar has__stack_smash_handler = LibcallImpls<(add __stack_smash_handler), isOSOpenBSD>; defvar has___guard_local = LibcallImpls<(add __guard_local), isOSOpenBSD>; @@ -1396,8 +1405,8 @@ defvar ExceptionModelCallsArm64EC = (add def WindowsARM64ECSystemLibrary : SystemRuntimeLibrary<isWindowsArm64EC, (add WinArm64ECDefaultRuntimeLibcallImpls, - arm64ec___stack_chk_fail, - __stack_chk_guard, + AvailableIf<arm64ec___stack_chk_fail, hasStackChkFail>, + AvailableIf<__stack_chk_guard, hasStackChkFail>, LibcallImpls<(add __security_check_cookie_arm64ec, __security_cookie), isWindowsMSVCEnvironment>, @@ -2318,11 +2327,11 @@ def PPCSystemLibrary LibmHasSinCosPPCF128, AvailableIf<memcpy, isNotAIX>, LibcallImpls<(add Int128RTLibcalls), isPPC64>, - has__stack_chk_fail, has__stack_smash_handler, has___guard_local, AvailableIf<__ssp_canary_word, isAIX>, - AvailableIf<__stack_chk_guard, isNotAIX>)>; + AvailableIf<__stack_chk_fail, isNotOSOpenBSD>, + AvailableIf<__stack_chk_guard, isNotOSAIXAndNotOSOpenBSD>)>; //===----------------------------------------------------------------------===// // RISCV Runtime Libcalls diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index d507ba2..581b4ad 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -62,6 +62,7 @@ LLVM_ABI void initializeBasicBlockSectionsPass(PassRegistry &); LLVM_ABI void initializeBarrierNoopPass(PassRegistry &); LLVM_ABI void initializeBasicAAWrapperPassPass(PassRegistry &); LLVM_ABI void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry &); +LLVM_ABI void initializeMachineBlockHashInfoPass(PassRegistry &); LLVM_ABI void initializeBranchFolderLegacyPass(PassRegistry &); LLVM_ABI void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry &); LLVM_ABI void initializeBranchRelaxationLegacyPass(PassRegistry &); diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 000472f..3a4dc5a 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -317,6 +317,8 @@ LLVM_ABI ThinBackend createInProcessThinBackend( /// distributor. /// RemoteCompiler specifies the path to a Clang executable to be invoked for /// the backend jobs. +/// RemoteCompilerPrependArgs specifies a list of prepend arguments to be +/// applied to the backend compilations. /// RemoteCompilerArgs specifies a list of arguments to be applied to the /// backend compilations. /// SaveTemps is a debugging tool that prevents temporary files created by this @@ -326,6 +328,7 @@ LLVM_ABI ThinBackend createOutOfProcessThinBackend( bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, StringRef LinkerOutputFile, StringRef Distributor, ArrayRef<StringRef> DistributorArgs, StringRef RemoteCompiler, + ArrayRef<StringRef> RemoteCompilerPrependArgs, ArrayRef<StringRef> RemoteCompilerArgs, bool SaveTemps); /// This ThinBackend writes individual module indexes to files, instead of @@ -459,6 +462,19 @@ private: ModuleMapType ModuleMap; // The bitcode modules to compile, if specified by the LTO Config. std::optional<ModuleMapType> ModulesToCompile; + + void setPrevailingModuleForGUID(GlobalValue::GUID GUID, StringRef Module) { + PrevailingModuleForGUID[GUID] = Module; + } + bool isPrevailingModuleForGUID(GlobalValue::GUID GUID, + StringRef Module) const { + auto It = PrevailingModuleForGUID.find(GUID); + return It != PrevailingModuleForGUID.end() && It->second == Module; + } + + private: + // Make this private so all accesses must go through above accessor methods + // to avoid inadvertently creating new entries on lookups. DenseMap<GlobalValue::GUID, StringRef> PrevailingModuleForGUID; } ThinLTO; diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h index 4a528ee..74abe34 100644 --- a/llvm/include/llvm/MC/MCContext.h +++ b/llvm/include/llvm/MC/MCContext.h @@ -175,7 +175,7 @@ private: unsigned GetInstance(unsigned LocalLabelVal); /// SHT_LLVM_BB_ADDR_MAP version to emit. - uint8_t BBAddrMapVersion = 3; + uint8_t BBAddrMapVersion = 4; /// The file name of the log file from the environment variable /// AS_SECURE_LOG_FILE. Which must be set before the .secure_log_unique diff --git a/llvm/include/llvm/Support/SpecialCaseList.h b/llvm/include/llvm/Support/SpecialCaseList.h index ead7655..a235975 100644 --- a/llvm/include/llvm/Support/SpecialCaseList.h +++ b/llvm/include/llvm/Support/SpecialCaseList.h @@ -13,7 +13,10 @@ #define LLVM_SUPPORT_SPECIALCASELIST_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/RadixTree.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/GlobPattern.h" @@ -162,6 +165,10 @@ private: }; std::vector<GlobMatcher::Glob> Globs; + + RadixTree<iterator_range<StringRef::const_iterator>, + SmallVector<const GlobMatcher::Glob *, 1>> + PrefixToGlob; }; /// Represents a set of patterns and their line numbers diff --git a/llvm/include/llvm/Support/Timer.h b/llvm/include/llvm/Support/Timer.h index 6a44758..527d67f 100644 --- a/llvm/include/llvm/Support/Timer.h +++ b/llvm/include/llvm/Support/Timer.h @@ -209,6 +209,7 @@ class TimerGroup { std::string Description; Timer *FirstTimer = nullptr; ///< First timer in the group. std::vector<PrintRecord> TimersToPrint; + bool PrintOnExit; TimerGroup **Prev; ///< Pointer to Next field of previous timergroup in list. TimerGroup *Next; ///< Pointer to next timergroup in list. @@ -217,13 +218,15 @@ class TimerGroup { friend class TimerGlobals; explicit TimerGroup(StringRef Name, StringRef Description, - sys::SmartMutex<true> &lock); + sys::SmartMutex<true> &lock, bool PrintOnExit); public: - LLVM_ABI explicit TimerGroup(StringRef Name, StringRef Description); + LLVM_ABI explicit TimerGroup(StringRef Name, StringRef Description, + bool PrintOnExit = true); LLVM_ABI explicit TimerGroup(StringRef Name, StringRef Description, - const StringMap<TimeRecord> &Records); + const StringMap<TimeRecord> &Records, + bool PrintOnExit = true); LLVM_ABI ~TimerGroup(); diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index 3d21f52..47d5d68 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -484,13 +484,6 @@ def propagate_undef_shuffle_mask: GICombineRule< [{ return Helper.matchUndefShuffleVectorMask(*${root}); }]), (apply [{ Helper.replaceInstWithUndef(*${root}); }])>; -// Replace a G_SHUFFLE_VECTOR with a G_EXTRACT_VECTOR_ELT. -def shuffle_to_extract: GICombineRule< - (defs root:$root), - (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, - [{ return Helper.matchShuffleToExtract(*${root}); }]), - (apply [{ Helper.applyShuffleToExtract(*${root}); }])>; - // Replace an insert/extract element of an out of bounds index with undef. def insert_extract_vec_elt_out_of_bounds : GICombineRule< (defs root:$root), @@ -1674,8 +1667,7 @@ def combine_shuffle_concat : GICombineRule< // Combines shuffles of vector into build_vector def combine_shuffle_vector_to_build_vector : GICombineRule< (defs root:$root), - (match (G_SHUFFLE_VECTOR $dst, $src1, $src2, $mask):$root, - [{ return Helper.matchCombineShuffleToBuildVector(*${root}); }]), + (match (G_SHUFFLE_VECTOR $dst, $src1, $src2, $mask):$root), (apply [{ Helper.applyCombineShuffleToBuildVector(*${root}); }])>; def insert_vector_element_idx_undef : GICombineRule< diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h index 8e83b046..7e68ad2 100644 --- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h +++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h @@ -115,9 +115,9 @@ struct ArchInfo { // Defines the following partial order, indicating when an architecture is // a superset of another: // - // v9.6a > v9.5a > v9.4a > v9.3a > v9.2a > v9.1a > v9a; - // v v v v v - // v8.9a > v8.8a > v8.7a > v8.6a > v8.5a > v8.4a > ... > v8a; + // v9.7a > v9.6a > v9.5a > v9.4a > v9.3a > v9.2a > v9.1a > v9a; + // v v v v v + // v8.9a > v8.8a > v8.7a > v8.6a > v8.5a > ... > v8a; // // v8r has no relation to anything. This is used to determine which // features to enable for a given architecture. See diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.def b/llvm/include/llvm/TargetParser/ARMTargetParser.def index ff53aa1..0ada2e7 100644 --- a/llvm/include/llvm/TargetParser/ARMTargetParser.def +++ b/llvm/include/llvm/TargetParser/ARMTargetParser.def @@ -187,6 +187,11 @@ ARM_ARCH("armv9.6-a", ARMV9_6A, "9.6-A", "+v9.6a", ARMBuildAttrs::CPUArch::v9_A, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS | ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM)) +ARM_ARCH("armv9.7-a", ARMV9_7A, "9.7-A", "+v9.7a", ARMBuildAttrs::CPUArch::v9_A, + FK_NEON_FP_ARMV8, + (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | + ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS | + ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM)) ARM_ARCH("armv8-r", ARMV8R, "8-R", "+v8r", ARMBuildAttrs::CPUArch::v8_R, FK_FPV5_SP_D16, (ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h index 5e43444..0e82dd2 100644 --- a/llvm/include/llvm/TargetParser/Triple.h +++ b/llvm/include/llvm/TargetParser/Triple.h @@ -115,6 +115,7 @@ public: enum SubArchType { NoSubArch, + ARMSubArch_v9_7a, ARMSubArch_v9_6a, ARMSubArch_v9_5a, ARMSubArch_v9_4a, |