diff options
Diffstat (limited to 'llvm/include')
120 files changed, 2345 insertions, 374 deletions
diff --git a/llvm/include/llvm/ADT/BitmaskEnum.h b/llvm/include/llvm/ADT/BitmaskEnum.h index d464cbc..9555fad 100644 --- a/llvm/include/llvm/ADT/BitmaskEnum.h +++ b/llvm/include/llvm/ADT/BitmaskEnum.h @@ -106,7 +106,7 @@ struct is_bitmask_enum : std::false_type {}; template <typename E> struct is_bitmask_enum< - E, std::enable_if_t<sizeof(E::LLVM_BITMASK_LARGEST_ENUMERATOR) >= 0>> + E, std::void_t<decltype(E::LLVM_BITMASK_LARGEST_ENUMERATOR)>> : std::true_type {}; /// Trait class to determine bitmask enumeration largest bit. @@ -114,7 +114,7 @@ template <typename E, typename Enable = void> struct largest_bitmask_enum_bit; template <typename E> struct largest_bitmask_enum_bit< - E, std::enable_if_t<sizeof(E::LLVM_BITMASK_LARGEST_ENUMERATOR) >= 0>> { + E, std::void_t<decltype(E::LLVM_BITMASK_LARGEST_ENUMERATOR)>> { using UnderlyingTy = std::underlying_type_t<E>; static constexpr UnderlyingTy value = static_cast<UnderlyingTy>(E::LLVM_BITMASK_LARGEST_ENUMERATOR); diff --git a/llvm/include/llvm/ADT/FoldingSet.h b/llvm/include/llvm/ADT/FoldingSet.h index 82a88c4..675b5c6 100644 --- a/llvm/include/llvm/ADT/FoldingSet.h +++ b/llvm/include/llvm/ADT/FoldingSet.h @@ -332,6 +332,14 @@ class FoldingSetNodeID { /// Use a SmallVector to avoid a heap allocation in the common case. SmallVector<unsigned, 32> Bits; + template <typename T> void AddIntegerImpl(T I) { + static_assert(std::is_integral_v<T> && sizeof(T) <= sizeof(unsigned) * 2, + "T must be an integer type no wider than 64 bits"); + Bits.push_back(static_cast<unsigned>(I)); + if constexpr (sizeof(unsigned) < sizeof(T)) + Bits.push_back(static_cast<unsigned long long>(I) >> 32); + } + public: FoldingSetNodeID() = default; @@ -348,24 +356,12 @@ public: "unexpected pointer size"); AddInteger(reinterpret_cast<uintptr_t>(Ptr)); } - void AddInteger(signed I) { Bits.push_back(I); } - void AddInteger(unsigned I) { Bits.push_back(I); } - void AddInteger(long I) { AddInteger((unsigned long)I); } - void AddInteger(unsigned long I) { - if (sizeof(long) == sizeof(int)) - AddInteger(unsigned(I)); - else if (sizeof(long) == sizeof(long long)) { - AddInteger((unsigned long long)I); - } else { - llvm_unreachable("unexpected sizeof(long)"); - } - } - void AddInteger(long long I) { AddInteger((unsigned long long)I); } - void AddInteger(unsigned long long I) { - AddInteger(unsigned(I)); - AddInteger(unsigned(I >> 32)); - } - + void AddInteger(signed I) { AddIntegerImpl(I); } + void AddInteger(unsigned I) { AddIntegerImpl(I); } + void AddInteger(long I) { AddIntegerImpl(I); } + void AddInteger(unsigned long I) { AddIntegerImpl(I); } + void AddInteger(long long I) { AddIntegerImpl(I); } + void AddInteger(unsigned long long I) { AddIntegerImpl(I); } void AddBoolean(bool B) { AddInteger(B ? 1U : 0U); } LLVM_ABI void AddString(StringRef String); LLVM_ABI void AddNodeID(const FoldingSetNodeID &ID); diff --git a/llvm/include/llvm/ADT/IndexedMap.h b/llvm/include/llvm/ADT/IndexedMap.h index cda0316..02193c7 100644 --- a/llvm/include/llvm/ADT/IndexedMap.h +++ b/llvm/include/llvm/ADT/IndexedMap.h @@ -22,52 +22,61 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/identity.h" #include <cassert> namespace llvm { -template <typename T, typename ToIndexT = identity<unsigned>> class IndexedMap { +namespace detail { +template <class Ty> struct IdentityIndex { + using argument_type = Ty; + + Ty &operator()(Ty &self) const { return self; } + const Ty &operator()(const Ty &self) const { return self; } +}; +} // namespace detail + +template <typename T, typename ToIndexT = detail::IdentityIndex<unsigned>> +class IndexedMap { using IndexT = typename ToIndexT::argument_type; // Prefer SmallVector with zero inline storage over std::vector. IndexedMaps // can grow very large and SmallVector grows more efficiently as long as T // is trivially copyable. using StorageT = SmallVector<T, 0>; - StorageT storage_; - T nullVal_; - ToIndexT toIndex_; + StorageT Storage; + T NullVal = T(); + ToIndexT ToIndex; public: - IndexedMap() : nullVal_(T()) {} + IndexedMap() = default; - explicit IndexedMap(const T &val) : nullVal_(val) {} + explicit IndexedMap(const T &Val) : NullVal(Val) {} - typename StorageT::reference operator[](IndexT n) { - assert(toIndex_(n) < storage_.size() && "index out of bounds!"); - return storage_[toIndex_(n)]; + typename StorageT::reference operator[](IndexT N) { + assert(ToIndex(N) < Storage.size() && "index out of bounds!"); + return Storage[ToIndex(N)]; } - typename StorageT::const_reference operator[](IndexT n) const { - assert(toIndex_(n) < storage_.size() && "index out of bounds!"); - return storage_[toIndex_(n)]; + typename StorageT::const_reference operator[](IndexT N) const { + assert(ToIndex(N) < Storage.size() && "index out of bounds!"); + return Storage[ToIndex(N)]; } - void reserve(typename StorageT::size_type s) { storage_.reserve(s); } + void reserve(typename StorageT::size_type S) { Storage.reserve(S); } - void resize(typename StorageT::size_type s) { storage_.resize(s, nullVal_); } + void resize(typename StorageT::size_type S) { Storage.resize(S, NullVal); } - void clear() { storage_.clear(); } + void clear() { Storage.clear(); } - void grow(IndexT n) { - unsigned NewSize = toIndex_(n) + 1; - if (NewSize > storage_.size()) + void grow(IndexT N) { + unsigned NewSize = ToIndex(N) + 1; + if (NewSize > Storage.size()) resize(NewSize); } - bool inBounds(IndexT n) const { return toIndex_(n) < storage_.size(); } + bool inBounds(IndexT N) const { return ToIndex(N) < Storage.size(); } - typename StorageT::size_type size() const { return storage_.size(); } + typename StorageT::size_type size() const { return Storage.size(); } }; } // namespace llvm diff --git a/llvm/include/llvm/ADT/STLForwardCompat.h b/llvm/include/llvm/ADT/STLForwardCompat.h index 9c81981..e02694f 100644 --- a/llvm/include/llvm/ADT/STLForwardCompat.h +++ b/llvm/include/llvm/ADT/STLForwardCompat.h @@ -125,7 +125,7 @@ struct detector<std::void_t<Op<Args...>>, Op, Args...> { template <template <class...> class Op, class... Args> using is_detected = typename detail::detector<void, Op, Args...>::value_t; -struct identity_cxx20 // NOLINT(readability-identifier-naming) +struct identity // NOLINT(readability-identifier-naming) { using is_transparent = void; diff --git a/llvm/include/llvm/ADT/SmallVector.h b/llvm/include/llvm/ADT/SmallVector.h index efae6f3..51109d1 100644 --- a/llvm/include/llvm/ADT/SmallVector.h +++ b/llvm/include/llvm/ADT/SmallVector.h @@ -14,6 +14,7 @@ #ifndef LLVM_ADT_SMALLVECTOR_H #define LLVM_ADT_SMALLVECTOR_H +#include "llvm/ADT/ADL.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/Support/Compiler.h" #include <algorithm> @@ -734,6 +735,12 @@ public: void assign(const SmallVectorImpl &RHS) { assign(RHS.begin(), RHS.end()); } + template <typename U, + typename = std::enable_if_t<std::is_convertible_v<U, T>>> + void assign(ArrayRef<U> AR) { + assign(AR.begin(), AR.end()); + } + iterator erase(const_iterator CI) { // Just cast away constness because this is a non-const member function. iterator I = const_cast<iterator>(CI); @@ -1228,7 +1235,7 @@ public: } template <typename U, - typename = std::enable_if_t<std::is_convertible<U, T>::value>> + typename = std::enable_if_t<std::is_convertible_v<U, T>>> explicit SmallVector(ArrayRef<U> A) : SmallVectorImpl<T>(N) { this->append(A.begin(), A.end()); } @@ -1289,28 +1296,27 @@ inline size_t capacity_in_bytes(const SmallVector<T, N> &X) { template <typename RangeType> using ValueTypeFromRangeType = - std::remove_const_t<std::remove_reference_t<decltype(*std::begin( - std::declval<RangeType &>()))>>; + std::remove_const_t<detail::ValueOfRange<RangeType>>; /// Given a range of type R, iterate the entire range and return a /// SmallVector with elements of the vector. This is useful, for example, /// when you want to iterate a range and then sort the results. template <unsigned Size, typename R> SmallVector<ValueTypeFromRangeType<R>, Size> to_vector(R &&Range) { - return {std::begin(Range), std::end(Range)}; + return {adl_begin(Range), adl_end(Range)}; } template <typename R> SmallVector<ValueTypeFromRangeType<R>> to_vector(R &&Range) { - return {std::begin(Range), std::end(Range)}; + return {adl_begin(Range), adl_end(Range)}; } template <typename Out, unsigned Size, typename R> SmallVector<Out, Size> to_vector_of(R &&Range) { - return {std::begin(Range), std::end(Range)}; + return {adl_begin(Range), adl_end(Range)}; } template <typename Out, typename R> SmallVector<Out> to_vector_of(R &&Range) { - return {std::begin(Range), std::end(Range)}; + return {adl_begin(Range), adl_end(Range)}; } // Explicit instantiations diff --git a/llvm/include/llvm/ADT/SparseMultiSet.h b/llvm/include/llvm/ADT/SparseMultiSet.h index 5e4e170..59de4cf 100644 --- a/llvm/include/llvm/ADT/SparseMultiSet.h +++ b/llvm/include/llvm/ADT/SparseMultiSet.h @@ -82,7 +82,7 @@ namespace llvm { /// @tparam SparseT An unsigned integer type. See above. /// template <typename ValueT, typename KeyT = unsigned, - typename KeyFunctorT = identity_cxx20, typename SparseT = uint8_t> + typename KeyFunctorT = identity, typename SparseT = uint8_t> class SparseMultiSet { static_assert(std::is_unsigned_v<SparseT>, "SparseT must be an unsigned integer type"); diff --git a/llvm/include/llvm/ADT/SparseSet.h b/llvm/include/llvm/ADT/SparseSet.h index 4697de09..41fd501 100644 --- a/llvm/include/llvm/ADT/SparseSet.h +++ b/llvm/include/llvm/ADT/SparseSet.h @@ -59,24 +59,20 @@ template <typename ValueT> struct SparseSetValTraits { } }; -/// SparseSetValFunctor - Helper class for selecting SparseSetValTraits. The -/// generic implementation handles ValueT classes which either provide -/// getSparseSetIndex() or specialize SparseSetValTraits<>. +/// SparseSetValFunctor - Helper class for getting a value's index. /// +/// In the generic case, this is done via SparseSetValTraits. When the value +/// type is the same as the key type, the KeyFunctor is used directly. template <typename KeyT, typename ValueT, typename KeyFunctorT> struct SparseSetValFunctor { unsigned operator()(const ValueT &Val) const { - return SparseSetValTraits<ValueT>::getValIndex(Val); + if constexpr (std::is_same_v<KeyT, ValueT>) + return KeyFunctorT()(Val); + else + return SparseSetValTraits<ValueT>::getValIndex(Val); } }; -/// SparseSetValFunctor<KeyT, KeyT> - Helper class for the common case of -/// identity key/value sets. -template <typename KeyT, typename KeyFunctorT> -struct SparseSetValFunctor<KeyT, KeyT, KeyFunctorT> { - unsigned operator()(const KeyT &Key) const { return KeyFunctorT()(Key); } -}; - /// SparseSet - Fast set implementation for objects that can be identified by /// small unsigned keys. /// @@ -117,7 +113,7 @@ struct SparseSetValFunctor<KeyT, KeyT, KeyFunctorT> { /// @tparam SparseT An unsigned integer type. See above. /// template <typename ValueT, typename KeyT = unsigned, - typename KeyFunctorT = identity_cxx20, typename SparseT = uint8_t> + typename KeyFunctorT = identity, typename SparseT = uint8_t> class SparseSet { static_assert(std::is_unsigned_v<SparseT>, "SparseT must be an unsigned integer type"); diff --git a/llvm/include/llvm/ADT/Twine.h b/llvm/include/llvm/ADT/Twine.h index d9f9c0f..e3b4d5e 100644 --- a/llvm/include/llvm/ADT/Twine.h +++ b/llvm/include/llvm/ADT/Twine.h @@ -285,7 +285,7 @@ public: } /// Construct from a StringRef. - /*implicit*/ Twine(const StringRef &Str) : LHSKind(PtrAndLengthKind) { + /*implicit*/ Twine(StringRef Str) : LHSKind(PtrAndLengthKind) { LHS.ptrAndLength.ptr = Str.data(); LHS.ptrAndLength.length = Str.size(); assert(isValid() && "Invalid twine!"); @@ -352,7 +352,7 @@ public: // right thing. Yet. /// Construct as the concatenation of a C string and a StringRef. - /*implicit*/ Twine(const char *LHS, const StringRef &RHS) + /*implicit*/ Twine(const char *LHS, StringRef RHS) : LHSKind(CStringKind), RHSKind(PtrAndLengthKind) { this->LHS.cString = LHS; this->RHS.ptrAndLength.ptr = RHS.data(); @@ -361,7 +361,7 @@ public: } /// Construct as the concatenation of a StringRef and a C string. - /*implicit*/ Twine(const StringRef &LHS, const char *RHS) + /*implicit*/ Twine(StringRef LHS, const char *RHS) : LHSKind(PtrAndLengthKind), RHSKind(CStringKind) { this->LHS.ptrAndLength.ptr = LHS.data(); this->LHS.ptrAndLength.length = LHS.size(); @@ -530,14 +530,14 @@ inline Twine operator+(const Twine &LHS, const Twine &RHS) { /// Additional overload to guarantee simplified codegen; this is equivalent to /// concat(). -inline Twine operator+(const char *LHS, const StringRef &RHS) { +inline Twine operator+(const char *LHS, StringRef RHS) { return Twine(LHS, RHS); } /// Additional overload to guarantee simplified codegen; this is equivalent to /// concat(). -inline Twine operator+(const StringRef &LHS, const char *RHS) { +inline Twine operator+(StringRef LHS, const char *RHS) { return Twine(LHS, RHS); } diff --git a/llvm/include/llvm/ADT/identity.h b/llvm/include/llvm/ADT/identity.h deleted file mode 100644 index 88d033f..0000000 --- a/llvm/include/llvm/ADT/identity.h +++ /dev/null @@ -1,31 +0,0 @@ -//===- llvm/ADT/Identity.h - Provide std::identity from C++20 ---*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file provides an implementation of std::identity from C++20. -// -// No library is required when using these functions. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_ADT_IDENTITY_H -#define LLVM_ADT_IDENTITY_H - -namespace llvm { - -// Similar to `std::identity` from C++20. -template <class Ty> struct identity { - using is_transparent = void; - using argument_type = Ty; - - Ty &operator()(Ty &self) const { return self; } - const Ty &operator()(const Ty &self) const { return self; } -}; - -} // namespace llvm - -#endif // LLVM_ADT_IDENTITY_H diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h index 58793ed..2832beb 100644 --- a/llvm/include/llvm/Analysis/InstSimplifyFolder.h +++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h @@ -120,7 +120,7 @@ public: } Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Type *Ty, - Instruction *FMFSource) const override { + Instruction *FMFSource = nullptr) const override { return simplifyBinaryIntrinsic(ID, Ty, LHS, RHS, SQ, dyn_cast_if_present<CallBase>(FMFSource)); } diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index 3d3ec14..04ea769 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -638,8 +638,12 @@ public: /// \p GEP The GEP. The indices contained in the GEP itself are ignored, /// instead we use IndexExprs. /// \p IndexExprs The expressions for the indices. - LLVM_ABI const SCEV * - getGEPExpr(GEPOperator *GEP, const SmallVectorImpl<const SCEV *> &IndexExprs); + LLVM_ABI const SCEV *getGEPExpr(GEPOperator *GEP, + ArrayRef<const SCEV *> IndexExprs); + LLVM_ABI const SCEV *getGEPExpr(const SCEV *BaseExpr, + ArrayRef<const SCEV *> IndexExprs, + Type *SrcElementTy, + GEPNoWrapFlags NW = GEPNoWrapFlags::none()); LLVM_ABI const SCEV *getAbsExpr(const SCEV *Op, bool IsNSW); LLVM_ABI const SCEV *getMinMaxExpr(SCEVTypes Kind, SmallVectorImpl<const SCEV *> &Operands); diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 5d3b233..7b7dc1b 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -227,6 +227,9 @@ public: /// Get the kind of extension that an instruction represents. LLVM_ABI static PartialReductionExtendKind getPartialReductionExtendKind(Instruction *I); + /// Get the kind of extension that a cast opcode represents. + LLVM_ABI static PartialReductionExtendKind + getPartialReductionExtendKind(Instruction::CastOps CastOpc); /// Construct a TTI object using a type implementing the \c Concept /// API below. diff --git a/llvm/include/llvm/AsmParser/AsmParserContext.h b/llvm/include/llvm/AsmParser/AsmParserContext.h new file mode 100644 index 0000000..1a397486 --- /dev/null +++ b/llvm/include/llvm/AsmParser/AsmParserContext.h @@ -0,0 +1,70 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ASMPARSER_ASMPARSERCONTEXT_H +#define LLVM_ASMPARSER_ASMPARSERCONTEXT_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/AsmParser/FileLoc.h" +#include "llvm/IR/Value.h" +#include <optional> + +namespace llvm { + +/// Registry of file location information for LLVM IR constructs. +/// +/// This class provides access to the file location information +/// for various LLVM IR constructs. Currently, it supports Function, +/// BasicBlock and Instruction locations. +/// +/// When available, it can answer queries about what is at a given +/// file location, as well as where in a file a given IR construct +/// is. +/// +/// This information is optionally emitted by the LLParser while +/// it reads LLVM textual IR. +class AsmParserContext { + DenseMap<Function *, FileLocRange> Functions; + DenseMap<BasicBlock *, FileLocRange> Blocks; + DenseMap<Instruction *, FileLocRange> Instructions; + +public: + std::optional<FileLocRange> getFunctionLocation(const Function *) const; + std::optional<FileLocRange> getBlockLocation(const BasicBlock *) const; + std::optional<FileLocRange> getInstructionLocation(const Instruction *) const; + /// Get the function at the requested location range. + /// If no single function occupies the queried range, or the record is + /// missing, a nullptr is returned. + Function *getFunctionAtLocation(const FileLocRange &) const; + /// Get the function at the requested location. + /// If no function occupies the queried location, or the record is missing, a + /// nullptr is returned. + Function *getFunctionAtLocation(const FileLoc &) const; + /// Get the block at the requested location range. + /// If no single block occupies the queried range, or the record is missing, a + /// nullptr is returned. + BasicBlock *getBlockAtLocation(const FileLocRange &) const; + /// Get the block at the requested location. + /// If no block occupies the queried location, or the record is missing, a + /// nullptr is returned. + BasicBlock *getBlockAtLocation(const FileLoc &) const; + /// Get the instruction at the requested location range. + /// If no single instruction occupies the queried range, or the record is + /// missing, a nullptr is returned. + Instruction *getInstructionAtLocation(const FileLocRange &) const; + /// Get the instruction at the requested location. + /// If no instruction occupies the queried location, or the record is missing, + /// a nullptr is returned. + Instruction *getInstructionAtLocation(const FileLoc &) const; + bool addFunctionLocation(Function *, const FileLocRange &); + bool addBlockLocation(BasicBlock *, const FileLocRange &); + bool addInstructionLocation(Instruction *, const FileLocRange &); +}; +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/AsmParser/FileLoc.h b/llvm/include/llvm/AsmParser/FileLoc.h new file mode 100644 index 0000000..02c1849 --- /dev/null +++ b/llvm/include/llvm/AsmParser/FileLoc.h @@ -0,0 +1,56 @@ +//===-- FileLoc.h ---------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ASMPARSER_FILELOC_H +#define LLVM_ASMPARSER_FILELOC_H + +#include <cassert> +#include <utility> + +namespace llvm { + +/// Struct holding Line:Column location +struct FileLoc { + /// 0-based line number + unsigned Line; + /// 0-based column number + unsigned Col; + + bool operator<=(const FileLoc &RHS) const { + return Line < RHS.Line || (Line == RHS.Line && Col <= RHS.Col); + } + + bool operator<(const FileLoc &RHS) const { + return Line < RHS.Line || (Line == RHS.Line && Col < RHS.Col); + } + + FileLoc(unsigned L, unsigned C) : Line(L), Col(C) {} + FileLoc(std::pair<unsigned, unsigned> LC) : Line(LC.first), Col(LC.second) {} +}; + +/// Struct holding a semiopen range [Start; End) +struct FileLocRange { + FileLoc Start; + FileLoc End; + + FileLocRange() : Start(0, 0), End(0, 0) {} + + FileLocRange(FileLoc S, FileLoc E) : Start(S), End(E) { + assert(Start <= End); + } + + bool contains(FileLoc L) const { return Start <= L && L < End; } + + bool contains(FileLocRange LR) const { + return Start <= LR.Start && LR.End <= End; + } +}; + +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/AsmParser/LLLexer.h b/llvm/include/llvm/AsmParser/LLLexer.h index 501a7ae..0e379e5 100644 --- a/llvm/include/llvm/AsmParser/LLLexer.h +++ b/llvm/include/llvm/AsmParser/LLLexer.h @@ -13,22 +13,25 @@ #ifndef LLVM_ASMPARSER_LLLEXER_H #define LLVM_ASMPARSER_LLLEXER_H -#include "LLToken.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/APSInt.h" +#include "llvm/AsmParser/LLToken.h" #include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" #include <string> namespace llvm { class Type; class SMDiagnostic; - class SourceMgr; class LLVMContext; class LLLexer { const char *CurPtr; StringRef CurBuf; + /// The end (exclusive) of the previous token. + const char *PrevTokEnd = nullptr; + enum class ErrorPriority { None, // No error message present. Parser, // Errors issued by parser. @@ -62,9 +65,7 @@ namespace llvm { explicit LLLexer(StringRef StartBuf, SourceMgr &SM, SMDiagnostic &, LLVMContext &C); - lltok::Kind Lex() { - return CurKind = LexToken(); - } + lltok::Kind Lex() { return CurKind = LexToken(); } typedef SMLoc LocTy; LocTy getLoc() const { return SMLoc::getFromPointer(TokStart); } @@ -79,6 +80,19 @@ namespace llvm { IgnoreColonInIdentifiers = val; } + /// Get the line, column position of the start of the current token, + /// zero-indexed + std::pair<unsigned, unsigned> getTokLineColumnPos() { + auto LC = SM.getLineAndColumn(SMLoc::getFromPointer(TokStart)); + return {LC.first - 1, LC.second - 1}; + } + /// Get the line, column position of the end of the previous token, + /// zero-indexed exclusive + std::pair<unsigned, unsigned> getPrevTokEndLineColumnPos() { + auto LC = SM.getLineAndColumn(SMLoc::getFromPointer(PrevTokEnd)); + return {LC.first - 1, LC.second - 1}; + } + // This returns true as a convenience for the parser functions that return // true on error. bool ParseError(LocTy ErrorLoc, const Twine &Msg) { diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h index c01de4a..9eb31d7 100644 --- a/llvm/include/llvm/AsmParser/LLParser.h +++ b/llvm/include/llvm/AsmParser/LLParser.h @@ -13,8 +13,9 @@ #ifndef LLVM_ASMPARSER_LLPARSER_H #define LLVM_ASMPARSER_LLPARSER_H -#include "LLLexer.h" #include "llvm/ADT/StringMap.h" +#include "llvm/AsmParser/AsmParserContext.h" +#include "llvm/AsmParser/LLLexer.h" #include "llvm/AsmParser/NumberedValues.h" #include "llvm/AsmParser/Parser.h" #include "llvm/IR/Attributes.h" @@ -177,6 +178,9 @@ namespace llvm { // Map of module ID to path. std::map<unsigned, StringRef> ModuleIdMap; + /// Keeps track of source locations for Values, BasicBlocks, and Functions. + AsmParserContext *ParserContext; + /// Only the llvm-as tool may set this to false to bypass /// UpgradeDebuginfo so it can generate broken bitcode. bool UpgradeDebugInfo; @@ -189,10 +193,11 @@ namespace llvm { public: LLParser(StringRef F, SourceMgr &SM, SMDiagnostic &Err, Module *M, ModuleSummaryIndex *Index, LLVMContext &Context, - SlotMapping *Slots = nullptr) + SlotMapping *Slots = nullptr, + AsmParserContext *ParserContext = nullptr) : Context(Context), OPLex(F, SM, Err, Context), Lex(F, SM, Err, Context), M(M), Index(Index), Slots(Slots), - BlockAddressPFS(nullptr) {} + BlockAddressPFS(nullptr), ParserContext(ParserContext) {} bool Run( bool UpgradeDebugInfo, DataLayoutCallbackTy DataLayoutCallback = [](StringRef, StringRef) { diff --git a/llvm/include/llvm/AsmParser/Parser.h b/llvm/include/llvm/AsmParser/Parser.h index c900b79..22b0881 100644 --- a/llvm/include/llvm/AsmParser/Parser.h +++ b/llvm/include/llvm/AsmParser/Parser.h @@ -15,6 +15,7 @@ #include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/AsmParser/AsmParserContext.h" #include "llvm/Support/Compiler.h" #include <memory> #include <optional> @@ -62,7 +63,8 @@ parseAssemblyFile(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context, /// parsing. LLVM_ABI std::unique_ptr<Module> parseAssemblyString(StringRef AsmString, SMDiagnostic &Err, - LLVMContext &Context, SlotMapping *Slots = nullptr); + LLVMContext &Context, SlotMapping *Slots = nullptr, + AsmParserContext *ParserContext = nullptr); /// Holds the Module and ModuleSummaryIndex returned by the interfaces /// that parse both. @@ -128,9 +130,9 @@ parseSummaryIndexAssemblyString(StringRef AsmString, SMDiagnostic &Err); LLVM_ABI std::unique_ptr<Module> parseAssembly( MemoryBufferRef F, SMDiagnostic &Err, LLVMContext &Context, SlotMapping *Slots = nullptr, - DataLayoutCallbackTy DataLayoutCallback = [](StringRef, StringRef) { - return std::nullopt; - }); + DataLayoutCallbackTy DataLayoutCallback = + [](StringRef, StringRef) { return std::nullopt; }, + AsmParserContext *ParserContext = nullptr); /// Parse LLVM Assembly including the summary index from a MemoryBuffer. /// @@ -169,9 +171,9 @@ parseSummaryIndexAssembly(MemoryBufferRef F, SMDiagnostic &Err); LLVM_ABI bool parseAssemblyInto( MemoryBufferRef F, Module *M, ModuleSummaryIndex *Index, SMDiagnostic &Err, SlotMapping *Slots = nullptr, - DataLayoutCallbackTy DataLayoutCallback = [](StringRef, StringRef) { - return std::nullopt; - }); + DataLayoutCallbackTy DataLayoutCallback = + [](StringRef, StringRef) { return std::nullopt; }, + AsmParserContext *ParserContext = nullptr); /// Parse a type and a constant value in the given string. /// diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index 8d0dc64..6ee6b666 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -1185,6 +1185,7 @@ enum : unsigned { SHT_LLVM_LTO = 0x6fff4c0c, // .llvm.lto for fat LTO. SHT_LLVM_JT_SIZES = 0x6fff4c0d, // LLVM jump tables sizes. SHT_LLVM_CFI_JUMP_TABLE = 0x6fff4c0e, // LLVM CFI jump table. + SHT_LLVM_CALL_GRAPH = 0x6fff4c0f, // LLVM Call Graph Section. // Android's experimental support for SHT_RELR sections. // https://android.googlesource.com/platform/bionic/+/b7feec74547f84559a1467aca02708ff61346d2a/libc/include/elf.h#512 SHT_ANDROID_RELR = 0x6fffff00, // Relocation entries; only offsets. diff --git a/llvm/include/llvm/CodeGen/AtomicExpand.h b/llvm/include/llvm/CodeGen/AtomicExpand.h index 1b8a988..34f520f 100644 --- a/llvm/include/llvm/CodeGen/AtomicExpand.h +++ b/llvm/include/llvm/CodeGen/AtomicExpand.h @@ -21,7 +21,7 @@ private: const TargetMachine *TM; public: - AtomicExpandPass(const TargetMachine *TM) : TM(TM) {} + AtomicExpandPass(const TargetMachine &TM) : TM(&TM) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h index 82dd5fe..48650a6 100644 --- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h +++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h @@ -155,7 +155,7 @@ class BasicBlockSectionsProfileReaderAnalysis public: static AnalysisKey Key; typedef BasicBlockSectionsProfileReader Result; - BasicBlockSectionsProfileReaderAnalysis(const TargetMachine *TM) : TM(TM) {} + BasicBlockSectionsProfileReaderAnalysis(const TargetMachine &TM) : TM(&TM) {} Result run(Function &F, FunctionAnalysisManager &AM); diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 4f27d9f1..76b6c8e 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -366,7 +366,7 @@ private: protected: explicit BasicTTIImplBase(const TargetMachine *TM, const DataLayout &DL) : BaseT(DL) {} - virtual ~BasicTTIImplBase() = default; + ~BasicTTIImplBase() override = default; using TargetTransformInfoImplBase::DL; @@ -821,13 +821,13 @@ public: SimplifyAndSetOp); } - virtual std::optional<unsigned> + std::optional<unsigned> getCacheSize(TargetTransformInfo::CacheLevel Level) const override { return std::optional<unsigned>( getST()->getCacheSize(static_cast<unsigned>(Level))); } - virtual std::optional<unsigned> + std::optional<unsigned> getCacheAssociativity(TargetTransformInfo::CacheLevel Level) const override { std::optional<unsigned> TargetResult = getST()->getCacheAssociativity(static_cast<unsigned>(Level)); @@ -838,31 +838,31 @@ public: return BaseT::getCacheAssociativity(Level); } - virtual unsigned getCacheLineSize() const override { + unsigned getCacheLineSize() const override { return getST()->getCacheLineSize(); } - virtual unsigned getPrefetchDistance() const override { + unsigned getPrefetchDistance() const override { return getST()->getPrefetchDistance(); } - virtual unsigned getMinPrefetchStride(unsigned NumMemAccesses, - unsigned NumStridedMemAccesses, - unsigned NumPrefetches, - bool HasCall) const override { + unsigned getMinPrefetchStride(unsigned NumMemAccesses, + unsigned NumStridedMemAccesses, + unsigned NumPrefetches, + bool HasCall) const override { return getST()->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses, NumPrefetches, HasCall); } - virtual unsigned getMaxPrefetchIterationsAhead() const override { + unsigned getMaxPrefetchIterationsAhead() const override { return getST()->getMaxPrefetchIterationsAhead(); } - virtual bool enableWritePrefetching() const override { + bool enableWritePrefetching() const override { return getST()->enableWritePrefetching(); } - virtual bool shouldPrefetchAddressSpace(unsigned AS) const override { + bool shouldPrefetchAddressSpace(unsigned AS) const override { return getST()->shouldPrefetchAddressSpace(AS); } diff --git a/llvm/include/llvm/CodeGen/CodeGenPrepare.h b/llvm/include/llvm/CodeGen/CodeGenPrepare.h index dee3a9e..e673d0f 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPrepare.h +++ b/llvm/include/llvm/CodeGen/CodeGenPrepare.h @@ -26,7 +26,7 @@ private: const TargetMachine *TM; public: - CodeGenPreparePass(const TargetMachine *TM) : TM(TM) {} + CodeGenPreparePass(const TargetMachine &TM) : TM(&TM) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; diff --git a/llvm/include/llvm/CodeGen/CodeGenTargetMachineImpl.h b/llvm/include/llvm/CodeGen/CodeGenTargetMachineImpl.h index 3950b95..7a6feda 100644 --- a/llvm/include/llvm/CodeGen/CodeGenTargetMachineImpl.h +++ b/llvm/include/llvm/CodeGen/CodeGenTargetMachineImpl.h @@ -42,7 +42,7 @@ public: /// Create a pass configuration object to be used by addPassToEmitX methods /// for generating a pipeline of CodeGen passes. - virtual TargetPassConfig *createPassConfig(PassManagerBase &PM) override; + TargetPassConfig *createPassConfig(PassManagerBase &PM) override; /// Add passes to the specified pass manager to get the specified file /// emitted. Typically this will involve several steps of code generation. diff --git a/llvm/include/llvm/CodeGen/CommandFlags.h b/llvm/include/llvm/CodeGen/CommandFlags.h index 39c5a8d..af66f2d 100644 --- a/llvm/include/llvm/CodeGen/CommandFlags.h +++ b/llvm/include/llvm/CodeGen/CommandFlags.h @@ -58,8 +58,6 @@ LLVM_ABI CodeGenFileType getFileType(); LLVM_ABI FramePointerKind getFramePointerUsage(); -LLVM_ABI bool getEnableUnsafeFPMath(); - LLVM_ABI bool getEnableNoInfsFPMath(); LLVM_ABI bool getEnableNoNaNsFPMath(); diff --git a/llvm/include/llvm/CodeGen/ComplexDeinterleavingPass.h b/llvm/include/llvm/CodeGen/ComplexDeinterleavingPass.h index 4383249..7b74f2a 100644 --- a/llvm/include/llvm/CodeGen/ComplexDeinterleavingPass.h +++ b/llvm/include/llvm/CodeGen/ComplexDeinterleavingPass.h @@ -24,10 +24,10 @@ class TargetMachine; struct ComplexDeinterleavingPass : public PassInfoMixin<ComplexDeinterleavingPass> { private: - TargetMachine *TM; + const TargetMachine *TM; public: - ComplexDeinterleavingPass(TargetMachine *TM) : TM(TM) {} + ComplexDeinterleavingPass(const TargetMachine &TM) : TM(&TM) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; diff --git a/llvm/include/llvm/CodeGen/DebugHandlerBase.h b/llvm/include/llvm/CodeGen/DebugHandlerBase.h index fee4bb1..e72801b 100644 --- a/llvm/include/llvm/CodeGen/DebugHandlerBase.h +++ b/llvm/include/llvm/CodeGen/DebugHandlerBase.h @@ -118,7 +118,7 @@ private: // AsmPrinterHandler overrides. public: - virtual ~DebugHandlerBase() override; + ~DebugHandlerBase() override; void beginModule(Module *M) override; diff --git a/llvm/include/llvm/CodeGen/DroppedVariableStatsMIR.h b/llvm/include/llvm/CodeGen/DroppedVariableStatsMIR.h index bc8dc1b..6da10d8 100644 --- a/llvm/include/llvm/CodeGen/DroppedVariableStatsMIR.h +++ b/llvm/include/llvm/CodeGen/DroppedVariableStatsMIR.h @@ -44,12 +44,11 @@ private: StringRef FuncOrModName); /// Override base class method to run on an llvm::MachineFunction /// specifically. - virtual void - visitEveryInstruction(unsigned &DroppedCount, - DenseMap<VarID, DILocation *> &InlinedAtsMap, - VarID Var) override; + void visitEveryInstruction(unsigned &DroppedCount, + DenseMap<VarID, DILocation *> &InlinedAtsMap, + VarID Var) override; /// Override base class method to run on DBG_VALUEs specifically. - virtual void visitEveryDebugRecord( + void visitEveryDebugRecord( DenseSet<VarID> &VarIDSet, DenseMap<StringRef, DenseMap<VarID, DILocation *>> &InlinedAtsMap, StringRef FuncName, bool Before) override; diff --git a/llvm/include/llvm/CodeGen/DwarfEHPrepare.h b/llvm/include/llvm/CodeGen/DwarfEHPrepare.h index 3f625cd..5b68b8c 100644 --- a/llvm/include/llvm/CodeGen/DwarfEHPrepare.h +++ b/llvm/include/llvm/CodeGen/DwarfEHPrepare.h @@ -24,7 +24,7 @@ class DwarfEHPreparePass : public PassInfoMixin<DwarfEHPreparePass> { const TargetMachine *TM; public: - explicit DwarfEHPreparePass(const TargetMachine *TM_) : TM(TM_) {} + explicit DwarfEHPreparePass(const TargetMachine &TM_) : TM(&TM_) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); }; diff --git a/llvm/include/llvm/CodeGen/ExpandFp.h b/llvm/include/llvm/CodeGen/ExpandFp.h index f1f441b..28e6aec 100644 --- a/llvm/include/llvm/CodeGen/ExpandFp.h +++ b/llvm/include/llvm/CodeGen/ExpandFp.h @@ -22,7 +22,7 @@ private: CodeGenOptLevel OptLevel; public: - explicit ExpandFpPass(const TargetMachine *TM, CodeGenOptLevel OptLevel); + explicit ExpandFpPass(const TargetMachine &TM, CodeGenOptLevel OptLevel); PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); static bool isRequired() { return true; } diff --git a/llvm/include/llvm/CodeGen/ExpandLargeDivRem.h b/llvm/include/llvm/CodeGen/ExpandLargeDivRem.h index 6fc4409..b73a382 100644 --- a/llvm/include/llvm/CodeGen/ExpandLargeDivRem.h +++ b/llvm/include/llvm/CodeGen/ExpandLargeDivRem.h @@ -20,7 +20,7 @@ private: const TargetMachine *TM; public: - explicit ExpandLargeDivRemPass(const TargetMachine *TM_) : TM(TM_) {} + explicit ExpandLargeDivRemPass(const TargetMachine &TM_) : TM(&TM_) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; diff --git a/llvm/include/llvm/CodeGen/ExpandMemCmp.h b/llvm/include/llvm/CodeGen/ExpandMemCmp.h index 94a8778..0b845e4 100644 --- a/llvm/include/llvm/CodeGen/ExpandMemCmp.h +++ b/llvm/include/llvm/CodeGen/ExpandMemCmp.h @@ -19,7 +19,7 @@ class ExpandMemCmpPass : public PassInfoMixin<ExpandMemCmpPass> { const TargetMachine *TM; public: - explicit ExpandMemCmpPass(const TargetMachine *TM_) : TM(TM_) {} + explicit ExpandMemCmpPass(const TargetMachine &TM_) : TM(&TM_) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); }; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h index ea3f1a8..6701ae0 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h @@ -40,14 +40,14 @@ public: // A CSE config for fully optimized builds. class LLVM_ABI CSEConfigFull : public CSEConfigBase { public: - virtual ~CSEConfigFull() = default; + ~CSEConfigFull() override = default; bool shouldCSEOpc(unsigned Opc) override; }; // Commonly used for O0 config. class LLVM_ABI CSEConfigConstantOnly : public CSEConfigBase { public: - virtual ~CSEConfigConstantOnly() = default; + ~CSEConfigConstantOnly() override = default; bool shouldCSEOpc(unsigned Opc) override; }; @@ -118,7 +118,7 @@ class LLVM_ABI GISelCSEInfo : public GISelChangeObserver { public: GISelCSEInfo() = default; - virtual ~GISelCSEInfo(); + ~GISelCSEInfo() override; void setMF(MachineFunction &MF); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h b/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h index 39ff90c..7a313f4 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h @@ -60,7 +60,7 @@ public: Combiner(MachineFunction &MF, CombinerInfo &CInfo, const TargetPassConfig *TPC, GISelValueTracking *VT, GISelCSEInfo *CSEInfo = nullptr); - virtual ~Combiner(); + ~Combiner() override; virtual bool tryCombineAll(MachineInstr &I) const = 0; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 93aff35..b0601eb 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -276,7 +276,6 @@ public: SmallVector<Register> &Ops) const; /// Replace \p MI with a build_vector. - bool matchCombineShuffleToBuildVector(MachineInstr &MI) const; void applyCombineShuffleToBuildVector(MachineInstr &MI) const; /// Try to combine G_SHUFFLE_VECTOR into G_CONCAT_VECTORS. @@ -295,8 +294,6 @@ public: /// Replace \p MI with a concat_vectors with \p Ops. void applyCombineShuffleVector(MachineInstr &MI, const ArrayRef<Register> Ops) const; - bool matchShuffleToExtract(MachineInstr &MI) const; - void applyShuffleToExtract(MachineInstr &MI) const; /// Optimize memcpy intrinsics et al, e.g. constant len calls. /// /p MaxLen if non-zero specifies the max length of a mem libcall to inline. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h index 2db66ba..17d656a 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h @@ -58,7 +58,7 @@ class LLVM_ABI GISelValueTracking : public GISelChangeObserver { public: GISelValueTracking(MachineFunction &MF, unsigned MaxDepth = 6); - ~GISelValueTracking() = default; + ~GISelValueTracking() override = default; const MachineFunction &getMachineFunction() const { return MF; } diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h index 3d7ccd5..268025e7 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -656,7 +656,7 @@ private: IRT->addSuccessorWithProb(Src, Dst, Prob); } - virtual ~GISelSwitchLowering() = default; + ~GISelSwitchLowering() override = default; private: IRTranslator *IRT; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h index cf65f34..5694079 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -21,7 +21,7 @@ class GISelObserverWrapper; class LLVM_ABI InstructionSelector : public GIMatchTableExecutor { public: - virtual ~InstructionSelector(); + ~InstructionSelector() override; /// Select the (possibly generic) instruction \p I to only use target-specific /// opcodes. It is OK to insert multiple instructions, but they cannot be diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index c0e426c..a458cbd 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -497,6 +497,7 @@ public: LLVM_ABI LegalizeResult lowerMinMax(MachineInstr &MI); LLVM_ABI LegalizeResult lowerFCopySign(MachineInstr &MI); LLVM_ABI LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI); + LLVM_ABI LegalizeResult lowerFMinimumMaximum(MachineInstr &MI); LLVM_ABI LegalizeResult lowerFMad(MachineInstr &MI); LLVM_ABI LegalizeResult lowerIntrinsicRound(MachineInstr &MI); LLVM_ABI LegalizeResult lowerFFloor(MachineInstr &MI); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h index b7ccfbb..8db99ba 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h @@ -210,8 +210,8 @@ struct SpecificConstantMatch { }; /// Matches a constant equal to \p RequestedValue. -inline SpecificConstantMatch m_SpecificICst(APInt RequestedValue) { - return SpecificConstantMatch(std::move(RequestedValue)); +inline SpecificConstantMatch m_SpecificICst(const APInt &RequestedValue) { + return SpecificConstantMatch(RequestedValue); } inline SpecificConstantMatch m_SpecificICst(int64_t RequestedValue) { @@ -221,7 +221,7 @@ inline SpecificConstantMatch m_SpecificICst(int64_t RequestedValue) { /// Matcher for a specific constant splat. struct SpecificConstantSplatMatch { APInt RequestedVal; - SpecificConstantSplatMatch(const APInt RequestedVal) + SpecificConstantSplatMatch(const APInt &RequestedVal) : RequestedVal(RequestedVal) {} bool match(const MachineRegisterInfo &MRI, Register Reg) { return isBuildVectorConstantSplat(Reg, MRI, RequestedVal, @@ -230,8 +230,9 @@ struct SpecificConstantSplatMatch { }; /// Matches a constant splat of \p RequestedValue. -inline SpecificConstantSplatMatch m_SpecificICstSplat(APInt RequestedValue) { - return SpecificConstantSplatMatch(std::move(RequestedValue)); +inline SpecificConstantSplatMatch +m_SpecificICstSplat(const APInt &RequestedValue) { + return SpecificConstantSplatMatch(RequestedValue); } inline SpecificConstantSplatMatch m_SpecificICstSplat(int64_t RequestedValue) { @@ -242,7 +243,7 @@ inline SpecificConstantSplatMatch m_SpecificICstSplat(int64_t RequestedValue) { /// Matcher for a specific constant or constant splat. struct SpecificConstantOrSplatMatch { APInt RequestedVal; - SpecificConstantOrSplatMatch(const APInt RequestedVal) + SpecificConstantOrSplatMatch(const APInt &RequestedVal) : RequestedVal(RequestedVal) {} bool match(const MachineRegisterInfo &MRI, Register Reg) { APInt MatchedVal; @@ -263,8 +264,8 @@ struct SpecificConstantOrSplatMatch { /// Matches a \p RequestedValue constant or a constant splat of \p /// RequestedValue. inline SpecificConstantOrSplatMatch -m_SpecificICstOrSplat(APInt RequestedValue) { - return SpecificConstantOrSplatMatch(std::move(RequestedValue)); +m_SpecificICstOrSplat(const APInt &RequestedValue) { + return SpecificConstantOrSplatMatch(RequestedValue); } inline SpecificConstantOrSplatMatch diff --git a/llvm/include/llvm/CodeGen/IndirectBrExpand.h b/llvm/include/llvm/CodeGen/IndirectBrExpand.h index f7d9d5d..572a712 100644 --- a/llvm/include/llvm/CodeGen/IndirectBrExpand.h +++ b/llvm/include/llvm/CodeGen/IndirectBrExpand.h @@ -19,7 +19,7 @@ class IndirectBrExpandPass : public PassInfoMixin<IndirectBrExpandPass> { const TargetMachine *TM; public: - IndirectBrExpandPass(const TargetMachine *TM) : TM(TM) {} + IndirectBrExpandPass(const TargetMachine &TM) : TM(&TM) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); }; diff --git a/llvm/include/llvm/CodeGen/InterleavedAccess.h b/llvm/include/llvm/CodeGen/InterleavedAccess.h index 31bd19a..42bfa84 100644 --- a/llvm/include/llvm/CodeGen/InterleavedAccess.h +++ b/llvm/include/llvm/CodeGen/InterleavedAccess.h @@ -25,7 +25,7 @@ class InterleavedAccessPass : public PassInfoMixin<InterleavedAccessPass> { const TargetMachine *TM; public: - explicit InterleavedAccessPass(const TargetMachine *TM) : TM(TM) {} + explicit InterleavedAccessPass(const TargetMachine &TM) : TM(&TM) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); }; diff --git a/llvm/include/llvm/CodeGen/InterleavedLoadCombine.h b/llvm/include/llvm/CodeGen/InterleavedLoadCombine.h index fa99aa3..2750fd4 100644 --- a/llvm/include/llvm/CodeGen/InterleavedLoadCombine.h +++ b/llvm/include/llvm/CodeGen/InterleavedLoadCombine.h @@ -20,7 +20,7 @@ class InterleavedLoadCombinePass const TargetMachine *TM; public: - explicit InterleavedLoadCombinePass(const TargetMachine *TM) : TM(TM) {} + explicit InterleavedLoadCombinePass(const TargetMachine &TM) : TM(&TM) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); }; diff --git a/llvm/include/llvm/CodeGen/MIR2Vec.h b/llvm/include/llvm/CodeGen/MIR2Vec.h index 7b1b5d9..44f009c 100644 --- a/llvm/include/llvm/CodeGen/MIR2Vec.h +++ b/llvm/include/llvm/CodeGen/MIR2Vec.h @@ -7,9 +7,20 @@ //===----------------------------------------------------------------------===// /// /// \file -/// This file defines the MIR2Vec vocabulary -/// analysis(MIR2VecVocabLegacyAnalysis), the core mir2vec::MIREmbedder -/// interface for generating Machine IR embeddings, and related utilities. +/// This file defines the MIR2Vec framework for generating Machine IR +/// embeddings. +/// +/// Design Overview: +/// ---------------------- +/// 1. MIR2VecVocabProvider - Core vocabulary loading logic (no PM dependency) +/// - Can be used standalone or wrapped by the pass manager +/// - Requires MachineModuleInfo with parsed machine functions +/// +/// 2. MIR2VecVocabLegacyAnalysis - Pass manager wrapper (ImmutablePass) +/// - Integrated and used by llc -print-mir2vec +/// +/// 3. MIREmbedder - Generates embeddings from vocabulary +/// - SymbolicMIREmbedder: MIR2Vec embedding implementation /// /// MIR2Vec extends IR2Vec to support Machine IR embeddings. It represents the /// LLVM Machine IR as embeddings which can be used as input to machine learning @@ -35,6 +46,8 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" @@ -52,11 +65,21 @@ class LLVMContext; class MIR2VecVocabLegacyAnalysis; class TargetInstrInfo; +enum class MIR2VecKind { Symbolic }; + namespace mir2vec { + +// Forward declarations +class MIREmbedder; +class SymbolicMIREmbedder; + extern llvm::cl::OptionCategory MIR2VecCategory; -extern cl::opt<float> OpcWeight; +extern cl::opt<float> OpcWeight, CommonOperandWeight, RegOperandWeight; using Embedding = ir2vec::Embedding; +using MachineInstEmbeddingsMap = DenseMap<const MachineInstr *, Embedding>; +using MachineBlockEmbeddingsMap = + DenseMap<const MachineBasicBlock *, Embedding>; /// Class for storing and accessing the MIR2Vec vocabulary. /// The MIRVocabulary class manages seed embeddings for LLVM Machine IR @@ -64,84 +87,326 @@ class MIRVocabulary { friend class llvm::MIR2VecVocabLegacyAnalysis; using VocabMap = std::map<std::string, ir2vec::Embedding>; -private: - // Define vocabulary layout - adapted for MIR + // MIRVocabulary Layout: + // +-------------------+-----------------------------------------------------+ + // | Entity Type | Description | + // +-------------------+-----------------------------------------------------+ + // | 1. Opcodes | Target specific opcodes derived from TII, grouped | + // | | by instruction semantics. | + // | 2. Common Operands| All common operand types, except register operands, | + // | | defined by MachineOperand::MachineOperandType enum. | + // | 3. Physical | Register classes defined by the target, specialized | + // | Reg classes | by physical registers. | + // | 4. Virtual | Register classes defined by the target, specialized | + // | Reg classes | by virtual and physical registers. | + // +-------------------+-----------------------------------------------------+ + + /// Layout information for the MIR vocabulary. Defines the starting index + /// and size of each section in the vocabulary. struct { size_t OpcodeBase = 0; - size_t OperandBase = 0; + size_t CommonOperandBase = 0; + size_t PhyRegBase = 0; + size_t VirtRegBase = 0; size_t TotalEntries = 0; } Layout; - enum class Section : unsigned { Opcodes = 0, MaxSections }; + // TODO: See if we can have only one reg classes section instead of physical + // and virtual separate sections in the vocabulary. This would reduce the + // number of vocabulary entities significantly. + // We can potentially distinguish physical and virtual registers by + // considering them as a separate feature. + enum class Section : unsigned { + Opcodes = 0, + CommonOperands = 1, + PhyRegisters = 2, + VirtRegisters = 3, + MaxSections + }; ir2vec::VocabStorage Storage; - mutable std::set<std::string> UniqueBaseOpcodeNames; + std::set<std::string> UniqueBaseOpcodeNames; + SmallVector<std::string, 24> RegisterOperandNames; + + // Some instructions have optional register operands that may be NoRegister. + // We return a zero vector in such cases. + Embedding ZeroEmbedding; + + // We have specialized MO_Register handling in the Register operand section, + // so we don't include it here. Also, no MO_DbgInstrRef for now. + static constexpr StringLiteral CommonOperandNames[] = { + "Immediate", "CImmediate", "FPImmediate", "MBB", + "FrameIndex", "ConstantPoolIndex", "TargetIndex", "JumpTableIndex", + "ExternalSymbol", "GlobalAddress", "BlockAddress", "RegisterMask", + "RegisterLiveOut", "Metadata", "MCSymbol", "CFIIndex", + "IntrinsicID", "Predicate", "ShuffleMask"}; + static_assert(std::size(CommonOperandNames) == MachineOperand::MO_Last - 1 && + "Common operand names size changed, update accordingly"); + const TargetInstrInfo &TII; - void generateStorage(const VocabMap &OpcodeMap); + const TargetRegisterInfo &TRI; + const MachineRegisterInfo &MRI; + + void generateStorage(const VocabMap &OpcodeMap, + const VocabMap &CommonOperandMap, + const VocabMap &PhyRegMap, const VocabMap &VirtRegMap); void buildCanonicalOpcodeMapping(); + void buildRegisterOperandMapping(); /// Get canonical index for a machine opcode unsigned getCanonicalOpcodeIndex(unsigned Opcode) const; + /// Get index for a common (non-register) machine operand + unsigned + getCommonOperandIndex(MachineOperand::MachineOperandType OperandType) const; + + /// Get index for a register machine operand + unsigned getRegisterOperandIndex(Register Reg) const; + + // Accessors for operand types + const Embedding & + operator[](MachineOperand::MachineOperandType OperandType) const { + unsigned LocalIndex = getCommonOperandIndex(OperandType); + return Storage[static_cast<unsigned>(Section::CommonOperands)][LocalIndex]; + } + + const Embedding &operator[](Register Reg) const { + // Reg is sometimes NoRegister (0) for optional operands. We return a zero + // vector in this case. + if (!Reg.isValid()) + return ZeroEmbedding; + // TODO: Implement proper stack slot handling for MIR2Vec embeddings. + // Stack slots represent frame indices and should have their own + // embedding strategy rather than defaulting to register class 0. + // Consider: 1) Separate vocabulary section for stack slots + // 2) Stack slot size/alignment based embeddings + // 3) Frame index based categorization + if (Reg.isStack()) + return ZeroEmbedding; + + unsigned LocalIndex = getRegisterOperandIndex(Reg); + auto SectionID = + Reg.isPhysical() ? Section::PhyRegisters : Section::VirtRegisters; + return Storage[static_cast<unsigned>(SectionID)][LocalIndex]; + } + + /// Get entity ID (flat index) for a common operand type + /// This is used for triplet generation + unsigned getEntityIDForCommonOperand( + MachineOperand::MachineOperandType OperandType) const { + return Layout.CommonOperandBase + getCommonOperandIndex(OperandType); + } + + /// Get entity ID (flat index) for a register + /// This is used for triplet generation + unsigned getEntityIDForRegister(Register Reg) const { + if (!Reg.isValid() || Reg.isStack()) + return Layout + .VirtRegBase; // Return VirtRegBase for invalid/stack registers + unsigned LocalIndex = getRegisterOperandIndex(Reg); + size_t BaseOffset = + Reg.isPhysical() ? Layout.PhyRegBase : Layout.VirtRegBase; + return BaseOffset + LocalIndex; + } + public: /// Static method for extracting base opcode names (public for testing) static std::string extractBaseOpcodeName(StringRef InstrName); - /// Get canonical index for base name (public for testing) + /// Get indices from opcode or operand names. These are public for testing. + /// String based lookups are inefficient and should be avoided in general. unsigned getCanonicalIndexForBaseName(StringRef BaseName) const; + unsigned getCanonicalIndexForOperandName(StringRef OperandName) const; + unsigned getCanonicalIndexForRegisterClass(StringRef RegName, + bool IsPhysical = true) const; /// Get the string key for a vocabulary entry at the given position std::string getStringKey(unsigned Pos) const; unsigned getDimension() const { return Storage.getDimension(); } + /// Get entity ID (flat index) for an opcode + /// This is used for triplet generation + unsigned getEntityIDForOpcode(unsigned Opcode) const { + return Layout.OpcodeBase + getCanonicalOpcodeIndex(Opcode); + } + + /// Get entity ID (flat index) for a machine operand + /// This is used for triplet generation + unsigned getEntityIDForMachineOperand(const MachineOperand &MO) const { + if (MO.getType() == MachineOperand::MO_Register) + return getEntityIDForRegister(MO.getReg()); + return getEntityIDForCommonOperand(MO.getType()); + } + // Accessor methods const Embedding &operator[](unsigned Opcode) const { unsigned LocalIndex = getCanonicalOpcodeIndex(Opcode); return Storage[static_cast<unsigned>(Section::Opcodes)][LocalIndex]; } + const Embedding &operator[](MachineOperand Operand) const { + auto OperandType = Operand.getType(); + if (OperandType == MachineOperand::MO_Register) + return operator[](Operand.getReg()); + else + return operator[](OperandType); + } + // Iterator access using const_iterator = ir2vec::VocabStorage::const_iterator; const_iterator begin() const { return Storage.begin(); } const_iterator end() const { return Storage.end(); } + MIRVocabulary() = delete; + + /// Factory method to create MIRVocabulary from vocabulary map + static Expected<MIRVocabulary> + create(VocabMap &&OpcMap, VocabMap &&CommonOperandsMap, VocabMap &&PhyRegMap, + VocabMap &&VirtRegMap, const TargetInstrInfo &TII, + const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI); + + /// Create a dummy vocabulary for testing purposes. + static Expected<MIRVocabulary> + createDummyVocabForTest(const TargetInstrInfo &TII, + const TargetRegisterInfo &TRI, + const MachineRegisterInfo &MRI, unsigned Dim = 1); + /// Total number of entries in the vocabulary size_t getCanonicalSize() const { return Storage.size(); } - MIRVocabulary() = delete; +private: + MIRVocabulary(VocabMap &&OpcMap, VocabMap &&CommonOperandsMap, + VocabMap &&PhyRegMap, VocabMap &&VirtRegMap, + const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, + const MachineRegisterInfo &MRI); +}; - /// Factory method to create MIRVocabulary from vocabulary map - static Expected<MIRVocabulary> create(VocabMap &&Entries, - const TargetInstrInfo &TII); +/// Base class for MIR embedders +class MIREmbedder { +protected: + const MachineFunction &MF; + const MIRVocabulary &Vocab; + + /// Dimension of the embeddings; Captured from the vocabulary + const unsigned Dimension; + + /// Weight for opcode embeddings + const float OpcWeight, CommonOperandWeight, RegOperandWeight; + + MIREmbedder(const MachineFunction &MF, const MIRVocabulary &Vocab) + : MF(MF), Vocab(Vocab), Dimension(Vocab.getDimension()), + OpcWeight(mir2vec::OpcWeight), + CommonOperandWeight(mir2vec::CommonOperandWeight), + RegOperandWeight(mir2vec::RegOperandWeight) {} + + /// Function to compute embeddings. + Embedding computeEmbeddings() const; + + /// Function to compute the embedding for a given machine basic block. + Embedding computeEmbeddings(const MachineBasicBlock &MBB) const; + + /// Function to compute the embedding for a given machine instruction. + /// Specific to the kind of embeddings being computed. + virtual Embedding computeEmbeddings(const MachineInstr &MI) const = 0; +public: + virtual ~MIREmbedder() = default; + + /// Factory method to create an Embedder object of the specified kind + /// Returns nullptr if the requested kind is not supported. + static std::unique_ptr<MIREmbedder> create(MIR2VecKind Mode, + const MachineFunction &MF, + const MIRVocabulary &Vocab); + + /// Computes and returns the embedding for a given machine instruction MI in + /// the machine function MF. + Embedding getMInstVector(const MachineInstr &MI) const { + return computeEmbeddings(MI); + } + + /// Computes and returns the embedding for a given machine basic block in the + /// machine function MF. + Embedding getMBBVector(const MachineBasicBlock &MBB) const { + return computeEmbeddings(MBB); + } + + /// Computes and returns the embedding for the current machine function. + Embedding getMFunctionVector() const { + // Currently, we always (re)compute the embeddings for the function. This is + // cheaper than caching the vector. + return computeEmbeddings(); + } +}; + +/// Class for computing Symbolic embeddings +/// Symbolic embeddings are constructed based on the entity-level +/// representations obtained from the MIR Vocabulary. +class SymbolicMIREmbedder : public MIREmbedder { private: - MIRVocabulary(VocabMap &&Entries, const TargetInstrInfo &TII); + Embedding computeEmbeddings(const MachineInstr &MI) const override; + +public: + SymbolicMIREmbedder(const MachineFunction &F, const MIRVocabulary &Vocab); + static std::unique_ptr<SymbolicMIREmbedder> + create(const MachineFunction &MF, const MIRVocabulary &Vocab); }; } // namespace mir2vec +/// MIR2Vec vocabulary provider used by pass managers and standalone tools. +/// This class encapsulates the core vocabulary loading logic and can be used +/// independently of the pass manager infrastructure. For pass-based usage, +/// see MIR2VecVocabLegacyAnalysis. +/// +/// Note: This provider pattern makes new PM migration straightforward when +/// needed. A new PM analysis wrapper can be added that delegates to this +/// provider, similar to how MIR2VecVocabLegacyAnalysis currently wraps it. +class MIR2VecVocabProvider { + using VocabMap = std::map<std::string, mir2vec::Embedding>; + +public: + MIR2VecVocabProvider(const MachineModuleInfo &MMI) : MMI(MMI) {} + + Expected<mir2vec::MIRVocabulary> getVocabulary(const Module &M); + +private: + Error readVocabulary(VocabMap &OpcVocab, VocabMap &CommonOperandVocab, + VocabMap &PhyRegVocabMap, VocabMap &VirtRegVocabMap); + const MachineModuleInfo &MMI; +}; + /// Pass to analyze and populate MIR2Vec vocabulary from a module class MIR2VecVocabLegacyAnalysis : public ImmutablePass { using VocabVector = std::vector<mir2vec::Embedding>; using VocabMap = std::map<std::string, mir2vec::Embedding>; - VocabMap StrVocabMap; - VocabVector Vocab; StringRef getPassName() const override; - Error readVocabulary(); protected: void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired<MachineModuleInfoWrapperPass>(); AU.setPreservesAll(); } + std::unique_ptr<MIR2VecVocabProvider> Provider; public: static char ID; MIR2VecVocabLegacyAnalysis() : ImmutablePass(ID) {} - Expected<mir2vec::MIRVocabulary> getMIR2VecVocabulary(const Module &M); + + Expected<mir2vec::MIRVocabulary> getMIR2VecVocabulary(const Module &M) { + MachineModuleInfo &MMI = + getAnalysis<MachineModuleInfoWrapperPass>().getMMI(); + if (!Provider) + Provider = std::make_unique<MIR2VecVocabProvider>(MMI); + return Provider->getVocabulary(M); + } + + MIR2VecVocabProvider &getProvider() { + assert(Provider && "Provider not initialized"); + return *Provider; + } }; /// This pass prints the embeddings in the MIR2Vec vocabulary @@ -166,6 +431,31 @@ public: } }; +/// This pass prints the MIR2Vec embeddings for machine functions, basic blocks, +/// and instructions +class MIR2VecPrinterLegacyPass : public MachineFunctionPass { + raw_ostream &OS; + +public: + static char ID; + explicit MIR2VecPrinterLegacyPass(raw_ostream &OS) + : MachineFunctionPass(ID), OS(OS) {} + + bool runOnMachineFunction(MachineFunction &MF) override; + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired<MIR2VecVocabLegacyAnalysis>(); + AU.setPreservesAll(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + StringRef getPassName() const override { + return "MIR2Vec Embedder Printer Pass"; + } +}; + +/// Create a machine pass that prints MIR2Vec embeddings +MachineFunctionPass *createMIR2VecPrinterLegacyPass(raw_ostream &OS); + } // namespace llvm -#endif // LLVM_CODEGEN_MIR2VEC_H
\ No newline at end of file +#endif // LLVM_CODEGEN_MIR2VEC_H diff --git a/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h b/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h new file mode 100644 index 0000000..d044d5f --- /dev/null +++ b/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h @@ -0,0 +1,114 @@ +//===- llvm/CodeGen/MachineBlockHashInfo.h ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Compute the hashes of basic blocks. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H +#define LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H + +#include "llvm/CodeGen/MachineFunctionPass.h" + +namespace llvm { + +/// An object wrapping several components of a basic block hash. The combined +/// (blended) hash is represented and stored as one uint64_t, while individual +/// components are of smaller size (e.g., uint16_t or uint8_t). +struct BlendedBlockHash { +public: + explicit BlendedBlockHash(uint16_t Offset, uint16_t OpcodeHash, + uint16_t InstrHash, uint16_t NeighborHash) + : Offset(Offset), OpcodeHash(OpcodeHash), InstrHash(InstrHash), + NeighborHash(NeighborHash) {} + + explicit BlendedBlockHash(uint64_t CombinedHash) { + Offset = CombinedHash & 0xffff; + CombinedHash >>= 16; + OpcodeHash = CombinedHash & 0xffff; + CombinedHash >>= 16; + InstrHash = CombinedHash & 0xffff; + CombinedHash >>= 16; + NeighborHash = CombinedHash & 0xffff; + } + + /// Combine the blended hash into uint64_t. + uint64_t combine() const { + uint64_t Hash = 0; + Hash |= uint64_t(NeighborHash); + Hash <<= 16; + Hash |= uint64_t(InstrHash); + Hash <<= 16; + Hash |= uint64_t(OpcodeHash); + Hash <<= 16; + Hash |= uint64_t(Offset); + return Hash; + } + + /// Compute a distance between two given blended hashes. The smaller the + /// distance, the more similar two blocks are. For identical basic blocks, + /// the distance is zero. + /// Since OpcodeHash is highly stable, we consider a match good only if + /// the OpcodeHashes are identical. Mismatched OpcodeHashes lead to low + /// matching accuracy, and poor matches undermine the quality of final + /// inference. Notably, during inference, we also consider the matching + /// ratio of basic blocks. For MachineFunctions with a low matching + /// ratio, we directly skip optimization to reduce the impact of + /// mismatches. This ensures even very poor profiles won’t cause negative + /// optimization. + /// In the context of matching, we consider NeighborHash to be more + /// important. This is especially true when accounting for inlining + /// scenarios, where the position of a basic block in the control + /// flow graph is more critical. + uint64_t distance(const BlendedBlockHash &BBH) const { + assert(OpcodeHash == BBH.OpcodeHash && + "incorrect blended hash distance computation"); + uint64_t Dist = 0; + // Account for NeighborHash + Dist += NeighborHash == BBH.NeighborHash ? 0 : 1; + Dist <<= 16; + // Account for InstrHash + Dist += InstrHash == BBH.InstrHash ? 0 : 1; + Dist <<= 16; + // Account for Offset + Dist += (Offset >= BBH.Offset ? Offset - BBH.Offset : BBH.Offset - Offset); + return Dist; + } + +private: + /// The offset of the basic block from the function start. + uint16_t Offset{0}; + /// Hash of the basic block instructions, excluding operands. + uint16_t OpcodeHash{0}; + /// Hash of the basic block instructions, including opcodes and + /// operands. + uint16_t InstrHash{0}; + /// OpcodeHash of the basic block together with OpcodeHashes of its + /// successors and predecessors. + uint16_t NeighborHash{0}; +}; + +class MachineBlockHashInfo : public MachineFunctionPass { + DenseMap<const MachineBasicBlock *, uint64_t> MBBHashInfo; + +public: + static char ID; + MachineBlockHashInfo(); + + StringRef getPassName() const override { return "Basic Block Hash Compute"; } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + + bool runOnMachineFunction(MachineFunction &F) override; + + uint64_t getMBBHash(const MachineBasicBlock &MBB); +}; + +} // end namespace llvm + +#endif // LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H diff --git a/llvm/include/llvm/CodeGen/MachineFunctionAnalysis.h b/llvm/include/llvm/CodeGen/MachineFunctionAnalysis.h index cd00e5f..bea1c78 100644 --- a/llvm/include/llvm/CodeGen/MachineFunctionAnalysis.h +++ b/llvm/include/llvm/CodeGen/MachineFunctionAnalysis.h @@ -42,7 +42,7 @@ public: FunctionAnalysisManager::Invalidator &); }; - MachineFunctionAnalysis(const TargetMachine *TM) : TM(TM) {}; + MachineFunctionAnalysis(const TargetMachine &TM) : TM(&TM) {}; LLVM_ABI Result run(Function &F, FunctionAnalysisManager &FAM); }; diff --git a/llvm/include/llvm/CodeGen/MachineModuleSlotTracker.h b/llvm/include/llvm/CodeGen/MachineModuleSlotTracker.h index 770f1b3..5504896 100644 --- a/llvm/include/llvm/CodeGen/MachineModuleSlotTracker.h +++ b/llvm/include/llvm/CodeGen/MachineModuleSlotTracker.h @@ -37,7 +37,7 @@ public: MachineModuleSlotTracker(const MachineModuleInfo &MMI, const MachineFunction *MF, bool ShouldInitializeAllMetadata = true); - ~MachineModuleSlotTracker(); + ~MachineModuleSlotTracker() override; void collectMachineMDNodes(MachineMDNodeListType &L) const; }; diff --git a/llvm/include/llvm/CodeGen/MachineOutliner.h b/llvm/include/llvm/CodeGen/MachineOutliner.h index fbb958cc..66cab3d 100644 --- a/llvm/include/llvm/CodeGen/MachineOutliner.h +++ b/llvm/include/llvm/CodeGen/MachineOutliner.h @@ -306,7 +306,7 @@ struct GlobalOutlinedFunction : public OutlinedFunction { } GlobalOutlinedFunction() = delete; - ~GlobalOutlinedFunction() = default; + ~GlobalOutlinedFunction() override = default; }; } // namespace outliner diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h index 272b4ac..9fddd47 100644 --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -69,6 +69,9 @@ LLVM_ABI MachineFunctionPass *createBasicBlockSectionsPass(); LLVM_ABI MachineFunctionPass *createBasicBlockPathCloningPass(); +/// createMachineBlockHashInfoPass - This pass computes basic block hashes. +LLVM_ABI MachineFunctionPass *createMachineBlockHashInfoPass(); + /// createMachineFunctionSplitterPass - This pass splits machine functions /// using profile information. LLVM_ABI MachineFunctionPass *createMachineFunctionSplitterPass(); @@ -93,6 +96,10 @@ createMachineFunctionPrinterPass(raw_ostream &OS, LLVM_ABI MachineFunctionPass * createMIR2VecVocabPrinterLegacyPass(raw_ostream &OS); +/// MIR2VecPrinter pass - This pass prints out the MIR2Vec embeddings for +/// machine functions, basic blocks and instructions. +LLVM_ABI MachineFunctionPass *createMIR2VecPrinterLegacyPass(raw_ostream &OS); + /// StackFramePrinter pass - This pass prints out the machine function's /// stack frame to the given stream as a debugging tool. LLVM_ABI MachineFunctionPass *createStackFrameLayoutAnalysisPass(); diff --git a/llvm/include/llvm/CodeGen/ResourcePriorityQueue.h b/llvm/include/llvm/CodeGen/ResourcePriorityQueue.h index c15bc67..0af4f47 100644 --- a/llvm/include/llvm/CodeGen/ResourcePriorityQueue.h +++ b/llvm/include/llvm/CodeGen/ResourcePriorityQueue.h @@ -75,7 +75,7 @@ namespace llvm { public: ResourcePriorityQueue(SelectionDAGISel *IS); - ~ResourcePriorityQueue(); + ~ResourcePriorityQueue() override; bool isBottomUp() const override { return false; } diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h index 201dc68..0dcf400 100644 --- a/llvm/include/llvm/CodeGen/SDPatternMatch.h +++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h @@ -559,6 +559,11 @@ m_VSelect(const T0_P &Cond, const T1_P &T, const T2_P &F) { } template <typename T0_P, typename T1_P, typename T2_P> +inline auto m_SelectLike(const T0_P &Cond, const T1_P &T, const T2_P &F) { + return m_AnyOf(m_Select(Cond, T, F), m_VSelect(Cond, T, F)); +} + +template <typename T0_P, typename T1_P, typename T2_P> inline Result_match<0, TernaryOpc_match<T0_P, T1_P, T2_P>> m_Load(const T0_P &Ch, const T1_P &Ptr, const T2_P &Offset) { return m_Result<0>( diff --git a/llvm/include/llvm/CodeGen/SafeStack.h b/llvm/include/llvm/CodeGen/SafeStack.h index e8f0d14..05ad40e 100644 --- a/llvm/include/llvm/CodeGen/SafeStack.h +++ b/llvm/include/llvm/CodeGen/SafeStack.h @@ -19,7 +19,7 @@ class SafeStackPass : public PassInfoMixin<SafeStackPass> { const TargetMachine *TM; public: - explicit SafeStackPass(const TargetMachine *TM_) : TM(TM_) {} + explicit SafeStackPass(const TargetMachine &TM_) : TM(&TM_) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); }; diff --git a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h index 4eacbdc..ab0d7e3 100644 --- a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -18,7 +18,6 @@ #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SparseMultiSet.h" -#include "llvm/ADT/identity.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineBasicBlock.h" @@ -90,7 +89,7 @@ namespace llvm { /// allocated once for the pass. It can be cleared in constant time and reused /// without any frees. using RegUnit2SUnitsMap = - SparseMultiSet<PhysRegSUOper, unsigned, identity_cxx20, uint16_t>; + SparseMultiSet<PhysRegSUOper, unsigned, identity, uint16_t>; /// Track local uses of virtual registers. These uses are gathered by the DAG /// builder and may be consulted by the scheduler to avoid iterating an entire diff --git a/llvm/include/llvm/CodeGen/SelectOptimize.h b/llvm/include/llvm/CodeGen/SelectOptimize.h index 37024a15..33f66bb 100644 --- a/llvm/include/llvm/CodeGen/SelectOptimize.h +++ b/llvm/include/llvm/CodeGen/SelectOptimize.h @@ -25,7 +25,7 @@ class SelectOptimizePass : public PassInfoMixin<SelectOptimizePass> { const TargetMachine *TM; public: - explicit SelectOptimizePass(const TargetMachine *TM) : TM(TM) {} + explicit SelectOptimizePass(const TargetMachine &TM) : TM(&TM) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); }; diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 69713d0..1759463 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -418,12 +418,21 @@ public: Unpredictable = 1 << 13, // Compare instructions which may carry the samesign flag. SameSign = 1 << 14, + // ISD::PTRADD operations that remain in bounds, i.e., the left operand is + // an address in a memory object in which the result of the operation also + // lies. WARNING: Since SDAG generally uses integers instead of pointer + // types, a PTRADD's pointer operand is effectively the result of an + // implicit inttoptr cast. Therefore, when an inbounds PTRADD uses a + // pointer P, transformations cannot assume that P has the provenance + // implied by its producer as, e.g, operations between producer and PTRADD + // that affect the provenance may have been optimized away. + InBounds = 1 << 15, // NOTE: Please update LargestValue in LLVM_DECLARE_ENUM_AS_BITMASK below // the class definition when adding new flags. PoisonGeneratingFlags = NoUnsignedWrap | NoSignedWrap | Exact | Disjoint | - NonNeg | NoNaNs | NoInfs | SameSign, + NonNeg | NoNaNs | NoInfs | SameSign | InBounds, FastMathFlags = NoNaNs | NoInfs | NoSignedZeros | AllowReciprocal | AllowContract | ApproximateFuncs | AllowReassociation, }; @@ -458,6 +467,7 @@ public: void setAllowReassociation(bool b) { setFlag<AllowReassociation>(b); } void setNoFPExcept(bool b) { setFlag<NoFPExcept>(b); } void setUnpredictable(bool b) { setFlag<Unpredictable>(b); } + void setInBounds(bool b) { setFlag<InBounds>(b); } // These are accessors for each flag. bool hasNoUnsignedWrap() const { return Flags & NoUnsignedWrap; } @@ -475,6 +485,7 @@ public: bool hasAllowReassociation() const { return Flags & AllowReassociation; } bool hasNoFPExcept() const { return Flags & NoFPExcept; } bool hasUnpredictable() const { return Flags & Unpredictable; } + bool hasInBounds() const { return Flags & InBounds; } bool operator==(const SDNodeFlags &Other) const { return Flags == Other.Flags; @@ -484,7 +495,7 @@ public: }; LLVM_DECLARE_ENUM_AS_BITMASK(decltype(SDNodeFlags::None), - SDNodeFlags::SameSign); + SDNodeFlags::InBounds); inline SDNodeFlags operator|(SDNodeFlags LHS, SDNodeFlags RHS) { LHS |= RHS; diff --git a/llvm/include/llvm/CodeGen/StackProtector.h b/llvm/include/llvm/CodeGen/StackProtector.h index dfafc78..fbe7935 100644 --- a/llvm/include/llvm/CodeGen/StackProtector.h +++ b/llvm/include/llvm/CodeGen/StackProtector.h @@ -86,7 +86,7 @@ class StackProtectorPass : public PassInfoMixin<StackProtectorPass> { const TargetMachine *TM; public: - explicit StackProtectorPass(const TargetMachine *TM) : TM(TM) {} + explicit StackProtectorPass(const TargetMachine &TM) : TM(&TM) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); }; diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 73f2c55..d6ed3a8 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -2127,7 +2127,7 @@ public: /// performs validation and error handling, returns the function. Otherwise, /// returns nullptr. Must be previously inserted by insertSSPDeclarations. /// Should be used only when getIRStackGuard returns nullptr. - virtual Function *getSSPStackGuardCheck(const Module &M) const; + Function *getSSPStackGuardCheck(const Module &M) const; protected: Value *getDefaultSafeStackPointerLocation(IRBuilderBase &IRB, @@ -2459,6 +2459,12 @@ public: return ISD::ANY_EXTEND; } + /// Returns how the platform's atomic rmw operations expect their input + /// argument to be extended (ZERO_EXTEND, SIGN_EXTEND, or ANY_EXTEND). + virtual ISD::NodeType getExtendForAtomicRMWArg(unsigned Op) const { + return ISD::ANY_EXTEND; + } + /// @} /// Returns true if we should normalize @@ -3570,6 +3576,10 @@ public: return nullptr; } + const RTLIB::RuntimeLibcallsInfo &getRuntimeLibcallsInfo() const { + return Libcalls; + } + void setLibcallImpl(RTLIB::Libcall Call, RTLIB::LibcallImpl Impl) { Libcalls.setLibcallImpl(Call, Impl); } @@ -3807,10 +3817,6 @@ private: /// The list of libcalls that the target will use. RTLIB::RuntimeLibcallsInfo Libcalls; - /// The ISD::CondCode that should be used to test the result of each of the - /// comparison libcall against zero. - ISD::CondCode CmpLibcallCCs[RTLIB::UNKNOWN_LIBCALL]; - /// The bits of IndexedModeActions used to store the legalisation actions /// We store the data as | ML | MS | L | S | each taking 4 bits. enum IndexedModeActionsBits { diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h index 822245f..f031353 100644 --- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h +++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h @@ -280,7 +280,7 @@ protected: unsigned Mode = 0); public: - virtual ~TargetRegisterInfo(); + ~TargetRegisterInfo() override; /// Return the number of registers for the function. (may overestimate) virtual unsigned getNumSupportedRegs(const MachineFunction &) const { diff --git a/llvm/include/llvm/CodeGen/TypePromotion.h b/llvm/include/llvm/CodeGen/TypePromotion.h index efe5823..ba32a21 100644 --- a/llvm/include/llvm/CodeGen/TypePromotion.h +++ b/llvm/include/llvm/CodeGen/TypePromotion.h @@ -26,7 +26,7 @@ private: const TargetMachine *TM; public: - TypePromotionPass(const TargetMachine *TM): TM(TM) { } + TypePromotionPass(const TargetMachine &TM) : TM(&TM) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; diff --git a/llvm/include/llvm/CodeGen/VLIWMachineScheduler.h b/llvm/include/llvm/CodeGen/VLIWMachineScheduler.h index 112ff6d..65ff1eb 100644 --- a/llvm/include/llvm/CodeGen/VLIWMachineScheduler.h +++ b/llvm/include/llvm/CodeGen/VLIWMachineScheduler.h @@ -223,7 +223,7 @@ public: enum { TopQID = 1, BotQID = 2, LogMaxQID = 2 }; ConvergingVLIWScheduler() : Top(TopQID, "TopQ"), Bot(BotQID, "BotQ") {} - virtual ~ConvergingVLIWScheduler() = default; + ~ConvergingVLIWScheduler() override = default; void initialize(ScheduleDAGMI *dag) override; diff --git a/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h b/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h index 3a36863..5a46207 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h +++ b/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h @@ -34,7 +34,7 @@ class LLVM_ABI AppendingTypeTableBuilder : public TypeCollection { public: explicit AppendingTypeTableBuilder(BumpPtrAllocator &Storage); - ~AppendingTypeTableBuilder(); + ~AppendingTypeTableBuilder() override; // TypeCollection overrides std::optional<TypeIndex> getFirst() override; diff --git a/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h b/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h index a587b3e..93e1c99 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h +++ b/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h @@ -47,7 +47,7 @@ class LLVM_ABI GlobalTypeTableBuilder : public TypeCollection { public: explicit GlobalTypeTableBuilder(BumpPtrAllocator &Storage); - ~GlobalTypeTableBuilder(); + ~GlobalTypeTableBuilder() override; // TypeCollection overrides std::optional<TypeIndex> getFirst() override; diff --git a/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h b/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h index 7302784..b9b2669 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h +++ b/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h @@ -43,7 +43,7 @@ class LLVM_ABI MergingTypeTableBuilder : public TypeCollection { public: explicit MergingTypeTableBuilder(BumpPtrAllocator &Storage); - ~MergingTypeTableBuilder(); + ~MergingTypeTableBuilder() override; // TypeCollection overrides std::optional<TypeIndex> getFirst() override; diff --git a/llvm/include/llvm/DebugInfo/GSYM/GsymContext.h b/llvm/include/llvm/DebugInfo/GSYM/GsymContext.h index 07d599c..e3e9b2b 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/GsymContext.h +++ b/llvm/include/llvm/DebugInfo/GSYM/GsymContext.h @@ -30,7 +30,7 @@ class GsymReader; class GsymContext : public DIContext { public: GsymContext(std::unique_ptr<GsymReader> Reader); - ~GsymContext(); + ~GsymContext() override; GsymContext(GsymContext &) = delete; GsymContext &operator=(GsymContext &) = delete; diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h index 0e7be45..34bace8 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h @@ -143,7 +143,7 @@ public: VirtualityCode(0) {} LVElement(const LVElement &) = delete; LVElement &operator=(const LVElement &) = delete; - virtual ~LVElement() = default; + ~LVElement() override = default; LVSubclassID getSubclassID() const { return SubclassID; } diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLine.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLine.h index 3618ce7..dd17f76 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLine.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLine.h @@ -53,7 +53,7 @@ public: } LVLine(const LVLine &) = delete; LVLine &operator=(const LVLine &) = delete; - virtual ~LVLine() = default; + ~LVLine() override = default; static bool classof(const LVElement *Element) { return Element->getSubclassID() == LVSubclassID::LV_LINE; @@ -117,7 +117,7 @@ public: LVLineDebug() : LVLine() { setIsLineDebug(); } LVLineDebug(const LVLineDebug &) = delete; LVLineDebug &operator=(const LVLineDebug &) = delete; - ~LVLineDebug() = default; + ~LVLineDebug() override = default; // Additional line information. It includes attributes that describes // states in the machine instructions (basic block, end prologue, etc). @@ -142,7 +142,7 @@ public: LVLineAssembler() : LVLine() { setIsLineAssembler(); } LVLineAssembler(const LVLineAssembler &) = delete; LVLineAssembler &operator=(const LVLineAssembler &) = delete; - ~LVLineAssembler() = default; + ~LVLineAssembler() override = default; // Print blanks as the line number. std::string noLineAsString(bool ShowZero) const override { diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h index 0718e33..090af54 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h @@ -100,7 +100,7 @@ public: LVLocation() : LVObject() { setIsLocation(); } LVLocation(const LVLocation &) = delete; LVLocation &operator=(const LVLocation &) = delete; - virtual ~LVLocation() = default; + ~LVLocation() override = default; PROPERTY(Property, IsAddressRange); PROPERTY(Property, IsBaseClassOffset); @@ -171,7 +171,7 @@ public: LVLocationSymbol() : LVLocation() {} LVLocationSymbol(const LVLocationSymbol &) = delete; LVLocationSymbol &operator=(const LVLocationSymbol &) = delete; - ~LVLocationSymbol() = default; + ~LVLocationSymbol() override = default; void addObject(LVAddress LowPC, LVAddress HighPC, LVUnsigned SectionOffset, uint64_t LocDescOffset) override; diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVRange.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVRange.h index b5c8333..4fa6a9a 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVRange.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVRange.h @@ -62,7 +62,7 @@ public: LVRange() : LVObject(), RangesTree(Allocator) {} LVRange(const LVRange &) = delete; LVRange &operator=(const LVRange &) = delete; - ~LVRange() = default; + ~LVRange() override = default; void addEntry(LVScope *Scope, LVAddress LowerAddress, LVAddress UpperAddress); void addEntry(LVScope *Scope); diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h index f4f3516..2e2619c 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h @@ -153,7 +153,7 @@ public: } LVScope(const LVScope &) = delete; LVScope &operator=(const LVScope &) = delete; - virtual ~LVScope() = default; + ~LVScope() override = default; static bool classof(const LVElement *Element) { return Element->getSubclassID() == LVSubclassID::LV_SCOPE; @@ -349,7 +349,7 @@ public: LVScopeAggregate() : LVScope() {} LVScopeAggregate(const LVScopeAggregate &) = delete; LVScopeAggregate &operator=(const LVScopeAggregate &) = delete; - ~LVScopeAggregate() = default; + ~LVScopeAggregate() override = default; // DW_AT_specification, DW_AT_abstract_origin. LVScope *getReference() const override { return Reference; } @@ -387,7 +387,7 @@ public: } LVScopeAlias(const LVScopeAlias &) = delete; LVScopeAlias &operator=(const LVScopeAlias &) = delete; - ~LVScopeAlias() = default; + ~LVScopeAlias() override = default; // Returns true if current scope is logically equal to the given 'Scope'. bool equals(const LVScope *Scope) const override; @@ -401,7 +401,7 @@ public: LVScopeArray() : LVScope() { setIsArray(); } LVScopeArray(const LVScopeArray &) = delete; LVScopeArray &operator=(const LVScopeArray &) = delete; - ~LVScopeArray() = default; + ~LVScopeArray() override = default; void resolveExtra() override; @@ -513,7 +513,7 @@ public: } LVScopeCompileUnit(const LVScopeCompileUnit &) = delete; LVScopeCompileUnit &operator=(const LVScopeCompileUnit &) = delete; - ~LVScopeCompileUnit() = default; + ~LVScopeCompileUnit() override = default; LVScope *getCompileUnitParent() const override { return static_cast<LVScope *>(const_cast<LVScopeCompileUnit *>(this)); @@ -643,7 +643,7 @@ public: LVScopeEnumeration() : LVScope() { setIsEnumeration(); } LVScopeEnumeration(const LVScopeEnumeration &) = delete; LVScopeEnumeration &operator=(const LVScopeEnumeration &) = delete; - ~LVScopeEnumeration() = default; + ~LVScopeEnumeration() override = default; // Returns true if current scope is logically equal to the given 'Scope'. bool equals(const LVScope *Scope) const override; @@ -658,7 +658,7 @@ public: LVScopeFormalPack() : LVScope() { setIsTemplatePack(); } LVScopeFormalPack(const LVScopeFormalPack &) = delete; LVScopeFormalPack &operator=(const LVScopeFormalPack &) = delete; - ~LVScopeFormalPack() = default; + ~LVScopeFormalPack() override = default; // Returns true if current scope is logically equal to the given 'Scope'. bool equals(const LVScope *Scope) const override; @@ -676,7 +676,7 @@ public: LVScopeFunction() : LVScope() {} LVScopeFunction(const LVScopeFunction &) = delete; LVScopeFunction &operator=(const LVScopeFunction &) = delete; - virtual ~LVScopeFunction() = default; + ~LVScopeFunction() override = default; // DW_AT_specification, DW_AT_abstract_origin. LVScope *getReference() const override { return Reference; } @@ -728,7 +728,7 @@ public: LVScopeFunctionInlined() : LVScopeFunction() { setIsInlinedFunction(); } LVScopeFunctionInlined(const LVScopeFunctionInlined &) = delete; LVScopeFunctionInlined &operator=(const LVScopeFunctionInlined &) = delete; - ~LVScopeFunctionInlined() = default; + ~LVScopeFunctionInlined() override = default; uint32_t getDiscriminator() const override { return Discriminator; } void setDiscriminator(uint32_t Value) override { @@ -767,7 +767,7 @@ public: LVScopeFunctionType() : LVScopeFunction() { setIsFunctionType(); } LVScopeFunctionType(const LVScopeFunctionType &) = delete; LVScopeFunctionType &operator=(const LVScopeFunctionType &) = delete; - ~LVScopeFunctionType() = default; + ~LVScopeFunctionType() override = default; void resolveExtra() override; }; @@ -781,7 +781,7 @@ public: } LVScopeModule(const LVScopeModule &) = delete; LVScopeModule &operator=(const LVScopeModule &) = delete; - ~LVScopeModule() = default; + ~LVScopeModule() override = default; // Returns true if current scope is logically equal to the given 'Scope'. bool equals(const LVScope *Scope) const override; @@ -797,7 +797,7 @@ public: LVScopeNamespace() : LVScope() { setIsNamespace(); } LVScopeNamespace(const LVScopeNamespace &) = delete; LVScopeNamespace &operator=(const LVScopeNamespace &) = delete; - ~LVScopeNamespace() = default; + ~LVScopeNamespace() override = default; // Access DW_AT_extension reference. LVScope *getReference() const override { return Reference; } @@ -827,7 +827,7 @@ public: LVScopeRoot() : LVScope() { setIsRoot(); } LVScopeRoot(const LVScopeRoot &) = delete; LVScopeRoot &operator=(const LVScopeRoot &) = delete; - ~LVScopeRoot() = default; + ~LVScopeRoot() override = default; StringRef getFileFormatName() const { return getStringPool().getString(FileFormatNameIndex); @@ -859,7 +859,7 @@ public: LVScopeTemplatePack() : LVScope() { setIsTemplatePack(); } LVScopeTemplatePack(const LVScopeTemplatePack &) = delete; LVScopeTemplatePack &operator=(const LVScopeTemplatePack &) = delete; - ~LVScopeTemplatePack() = default; + ~LVScopeTemplatePack() override = default; // Returns true if current scope is logically equal to the given 'Scope'. bool equals(const LVScope *Scope) const override; diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h index ec9017e..c5314fc 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h @@ -74,7 +74,7 @@ public: } LVSymbol(const LVSymbol &) = delete; LVSymbol &operator=(const LVSymbol &) = delete; - ~LVSymbol() = default; + ~LVSymbol() override = default; static bool classof(const LVElement *Element) { return Element->getSubclassID() == LVSubclassID::LV_SYMBOL; diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVType.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVType.h index 59e6a92..af4abcf 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVType.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVType.h @@ -67,7 +67,7 @@ public: LVType() : LVElement(LVSubclassID::LV_TYPE) { setIsType(); } LVType(const LVType &) = delete; LVType &operator=(const LVType &) = delete; - virtual ~LVType() = default; + ~LVType() override = default; static bool classof(const LVElement *Element) { return Element->getSubclassID() == LVSubclassID::LV_TYPE; @@ -157,7 +157,7 @@ public: } LVTypeDefinition(const LVTypeDefinition &) = delete; LVTypeDefinition &operator=(const LVTypeDefinition &) = delete; - ~LVTypeDefinition() = default; + ~LVTypeDefinition() override = default; // Return the underlying type for a type definition. LVElement *getUnderlyingType() override; @@ -183,7 +183,7 @@ public: } LVTypeEnumerator(const LVTypeEnumerator &) = delete; LVTypeEnumerator &operator=(const LVTypeEnumerator &) = delete; - ~LVTypeEnumerator() = default; + ~LVTypeEnumerator() override = default; // Process the values for a DW_TAG_enumerator. StringRef getValue() const override { @@ -206,7 +206,7 @@ public: LVTypeImport() : LVType() { setIncludeInPrint(); } LVTypeImport(const LVTypeImport &) = delete; LVTypeImport &operator=(const LVTypeImport &) = delete; - ~LVTypeImport() = default; + ~LVTypeImport() override = default; // Returns true if current type is logically equal to the given 'Type'. bool equals(const LVType *Type) const override; @@ -223,7 +223,7 @@ public: LVTypeParam(); LVTypeParam(const LVTypeParam &) = delete; LVTypeParam &operator=(const LVTypeParam &) = delete; - ~LVTypeParam() = default; + ~LVTypeParam() override = default; // Template parameter value. StringRef getValue() const override { @@ -256,7 +256,7 @@ public: } LVTypeSubrange(const LVTypeSubrange &) = delete; LVTypeSubrange &operator=(const LVTypeSubrange &) = delete; - ~LVTypeSubrange() = default; + ~LVTypeSubrange() override = default; int64_t getCount() const override { return getIsSubrangeCount() ? LowerBound : 0; diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h index 2cf4a8e..cc8dda2 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h @@ -192,7 +192,7 @@ public: : LVReader(Filename, FileFormatName, W, BinaryType) {} LVBinaryReader(const LVBinaryReader &) = delete; LVBinaryReader &operator=(const LVBinaryReader &) = delete; - virtual ~LVBinaryReader() = default; + ~LVBinaryReader() override = default; void addInlineeLines(LVScope *Scope, LVLines &Lines) { CUInlineeLines.emplace(Scope, std::make_unique<LVLines>(std::move(Lines))); diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h index 4dd7c96..9f6fd553 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h @@ -200,7 +200,7 @@ public: Input(&Pdb), ExePath(ExePath), LogicalVisitor(this, W, Input) {} LVCodeViewReader(const LVCodeViewReader &) = delete; LVCodeViewReader &operator=(const LVCodeViewReader &) = delete; - ~LVCodeViewReader() = default; + ~LVCodeViewReader() override = default; void getLinkageName(const llvm::object::coff_section *CoffSection, uint32_t RelocOffset, uint32_t Offset, diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h index 2abc18b..1cf2914 100644 --- a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h +++ b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h @@ -123,7 +123,7 @@ public: Obj(Obj) {} LVDWARFReader(const LVDWARFReader &) = delete; LVDWARFReader &operator=(const LVDWARFReader &) = delete; - ~LVDWARFReader() = default; + ~LVDWARFReader() override = default; LVAddress getCUBaseAddress() const { return CUBaseAddress; } void setCUBaseAddress(LVAddress Address) { CUBaseAddress = Address; } diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h index b21cd09..c0b3151 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h @@ -20,7 +20,7 @@ namespace pdb { class LLVM_ABI PDBSymbolTypeBuiltin : public PDBSymbol { DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::BuiltinType) public: - ~PDBSymbolTypeBuiltin(); + ~PDBSymbolTypeBuiltin() override; void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getBuiltinType) diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h index 8613ddd..f05febf 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h @@ -448,7 +448,7 @@ public: FailedToMaterialize(std::shared_ptr<SymbolStringPool> SSP, std::shared_ptr<SymbolDependenceMap> Symbols); - ~FailedToMaterialize(); + ~FailedToMaterialize() override; std::error_code convertToErrorCode() const override; void log(raw_ostream &OS) const override; const SymbolDependenceMap &getSymbols() const { return *Symbols; } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h b/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h index 254b897..e84eb4b 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h @@ -70,7 +70,7 @@ public: DebugObjectManagerPlugin(ExecutionSession &ES, std::unique_ptr<DebugObjectRegistrar> Target, bool RequireDebugSections, bool AutoRegisterCode); - ~DebugObjectManagerPlugin(); + ~DebugObjectManagerPlugin() override; void notifyMaterializing(MaterializationResponsibility &MR, jitlink::LinkGraph &G, jitlink::JITLinkContext &Ctx, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h b/llvm/include/llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h index 179fedc..f9b1d2c 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h @@ -32,7 +32,7 @@ public: ExecutorAddr RegisterPerfEndAddr, ExecutorAddr RegisterPerfImplAddr, bool EmitDebugInfo, bool EmitUnwindInfo); - ~PerfSupportPlugin(); + ~PerfSupportPlugin() override; void modifyPassConfig(MaterializationResponsibility &MR, jitlink::LinkGraph &G, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h index fa48480..031bb27 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h @@ -52,7 +52,7 @@ public: EPCGenericRTDyldMemoryManager(EPCGenericRTDyldMemoryManager &&) = delete; EPCGenericRTDyldMemoryManager & operator=(EPCGenericRTDyldMemoryManager &&) = delete; - ~EPCGenericRTDyldMemoryManager(); + ~EPCGenericRTDyldMemoryManager() override; uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, unsigned SectionID, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index fecffc2..fa4bb9d 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -285,7 +285,7 @@ public: /// Map type for initializing the manager. See init. using StubInitsMap = StringMap<std::pair<ExecutorAddr, JITSymbolFlags>>; - virtual ~IndirectStubsManager() = default; + ~IndirectStubsManager() override = default; /// Create a single stub with the given name, target address and flags. virtual Error createStub(StringRef StubName, ExecutorAddr StubAddr, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Layer.h b/llvm/include/llvm/ExecutionEngine/Orc/Layer.h index 8dfc12e..25df380 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Layer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Layer.h @@ -136,7 +136,7 @@ public: static char ID; ObjectLayer(ExecutionSession &ES); - virtual ~ObjectLayer(); + ~ObjectLayer() override; /// Returns the execution session for this layer. ExecutionSession &getExecutionSession() { return ES; } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h index d3643f9..2079a35 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h @@ -88,7 +88,7 @@ public: std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr); /// Destroy the LinkGraphLinkingLayer. - ~LinkGraphLinkingLayer(); + ~LinkGraphLinkingLayer() override; /// Add a plugin. LinkGraphLinkingLayer &addPlugin(std::shared_ptr<Plugin> P) { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index 9c34bf1..b23093d 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -286,7 +286,6 @@ private: // FIXME: ObjCImageInfos and HeaderAddrs need to be cleared when // JITDylibs are removed. DenseMap<JITDylib *, ObjCImageInfo> ObjCImageInfos; - DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs; }; using GetJITDylibHeaderSendResultFn = diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h index 1fb472a1..8c6a8f5 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h @@ -58,7 +58,7 @@ public: RTDyldObjectLinkingLayer(ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager); - ~RTDyldObjectLinkingLayer(); + ~RTDyldObjectLinkingLayer() override; /// Emit the object. void emit(std::unique_ptr<MaterializationResponsibility> R, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h index 4a32113b..6adaa8a 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h @@ -34,7 +34,7 @@ using ExecutorAddrDiff = uint64_t; class ExecutorAddr { public: /// A wrap/unwrap function that leaves pointers unmodified. - using rawPtr = llvm::identity_cxx20; + using rawPtr = llvm::identity; #if __has_feature(ptrauth_calls) template <typename T> class PtrauthSignDefault { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/SymbolFilter.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/SymbolFilter.h new file mode 100644 index 0000000..5170893 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/SymbolFilter.h @@ -0,0 +1,173 @@ +//===- SymbolFilter.h - Utilities for Symbol Filtering ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_SYMBOLFILTER_H +#define LLVM_EXECUTIONENGINE_ORC_SHARED_SYMBOLFILTER_H + +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" + +#include <cmath> +#include <type_traits> +#include <vector> + +namespace llvm { +namespace orc { + +namespace shared { +using SPSBloomFilter = + SPSTuple<bool, uint32_t, uint32_t, uint32_t, SPSSequence<uint64_t>>; +} + +class BloomFilter { +public: + using HashFunc = std::function<uint32_t(StringRef)>; + + BloomFilter() = default; + BloomFilter(BloomFilter &&) noexcept = default; + BloomFilter &operator=(BloomFilter &&) noexcept = default; + BloomFilter(const BloomFilter &) = delete; + BloomFilter &operator=(const BloomFilter &) = delete; + + BloomFilter(uint32_t SymbolCount, float FalsePositiveRate, HashFunc hashFn) + : HashFn(std::move(hashFn)) { + initialize(SymbolCount, FalsePositiveRate); + } + bool isInitialized() const { return Initialized; } + + void add(StringRef Sym) { + assert(Initialized); + addHash(HashFn(Sym)); + } + + bool mayContain(StringRef Sym) const { + return !isEmpty() && testHash(HashFn(Sym)); + } + + bool isEmpty() const { return SymbolCount == 0; } + +private: + friend class shared::SPSSerializationTraits<shared::SPSBloomFilter, + BloomFilter>; + static constexpr uint32_t BitsPerEntry = 64; + + bool Initialized = false; + uint32_t SymbolCount = 0; + uint32_t BloomSize = 0; + uint32_t BloomShift = 0; + std::vector<uint64_t> BloomTable; + HashFunc HashFn; + + void initialize(uint32_t SymCount, float FalsePositiveRate) { + assert(SymCount > 0); + SymbolCount = SymCount; + Initialized = true; + + float ln2 = std::log(2.0f); + float M = -1.0f * SymbolCount * std::log(FalsePositiveRate) / (ln2 * ln2); + BloomSize = static_cast<uint32_t>(std::ceil(M / BitsPerEntry)); + BloomShift = std::min(6u, log2ceil(SymbolCount)); + BloomTable.resize(BloomSize, 0); + } + + void addHash(uint32_t Hash) { + uint32_t Hash2 = Hash >> BloomShift; + uint32_t N = (Hash / BitsPerEntry) % BloomSize; + uint64_t Mask = + (1ULL << (Hash % BitsPerEntry)) | (1ULL << (Hash2 % BitsPerEntry)); + BloomTable[N] |= Mask; + } + + bool testHash(uint32_t Hash) const { + uint32_t Hash2 = Hash >> BloomShift; + uint32_t N = (Hash / BitsPerEntry) % BloomSize; + uint64_t Mask = + (1ULL << (Hash % BitsPerEntry)) | (1ULL << (Hash2 % BitsPerEntry)); + return (BloomTable[N] & Mask) == Mask; + } + + static constexpr uint32_t log2ceil(uint32_t V) { + return V <= 1 ? 0 : 32 - countl_zero(V - 1); + } +}; + +class BloomFilterBuilder { +public: + using HashFunc = BloomFilter::HashFunc; + + BloomFilterBuilder() = default; + + BloomFilterBuilder &setFalsePositiveRate(float Rate) { + assert(Rate > 0.0f && Rate < 1.0f); + FalsePositiveRate = Rate; + return *this; + } + + BloomFilterBuilder &setHashFunction(HashFunc Fn) { + HashFn = std::move(Fn); + return *this; + } + + BloomFilter build(ArrayRef<StringRef> Symbols) const { + assert(!Symbols.empty() && "Cannot build filter from empty symbol list."); + BloomFilter F(static_cast<uint32_t>(Symbols.size()), FalsePositiveRate, + HashFn); + for (const auto &Sym : Symbols) + F.add(Sym); + + return F; + } + +private: + float FalsePositiveRate = 0.02f; + HashFunc HashFn = [](StringRef S) -> uint32_t { + uint32_t H = 5381; + for (char C : S) + H = ((H << 5) + H) + static_cast<uint8_t>(C); // H * 33 + C + return H; + }; +}; + +namespace shared { + +template <> class SPSSerializationTraits<SPSBloomFilter, BloomFilter> { +public: + static size_t size(const BloomFilter &Filter) { + return SPSBloomFilter::AsArgList::size( + Filter.Initialized, Filter.SymbolCount, Filter.BloomSize, + Filter.BloomShift, Filter.BloomTable); + } + + static bool serialize(SPSOutputBuffer &OB, const BloomFilter &Filter) { + return SPSBloomFilter::AsArgList::serialize( + OB, Filter.Initialized, Filter.SymbolCount, Filter.BloomSize, + Filter.BloomShift, Filter.BloomTable); + } + + static bool deserialize(SPSInputBuffer &IB, BloomFilter &Filter) { + bool IsInitialized; + uint32_t SymbolCount = 0, BloomSize = 0, BloomShift = 0; + std::vector<uint64_t> BloomTable; + + if (!SPSBloomFilter::AsArgList::deserialize( + IB, IsInitialized, SymbolCount, BloomSize, BloomShift, BloomTable)) + return false; + + Filter.Initialized = IsInitialized; + Filter.SymbolCount = SymbolCount; + Filter.BloomSize = BloomSize; + Filter.BloomShift = BloomShift; + Filter.BloomTable = std::move(BloomTable); + + return true; + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm +#endif // LLVM_EXECUTIONENGINE_ORC_SHARED_SYMBOLFILTER_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h b/llvm/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h index 7acb6a4..8176183 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h @@ -69,7 +69,7 @@ public: SimpleRemoteEPC &operator=(const SimpleRemoteEPC &) = delete; SimpleRemoteEPC(SimpleRemoteEPC &&) = delete; SimpleRemoteEPC &operator=(SimpleRemoteEPC &&) = delete; - ~SimpleRemoteEPC(); + ~SimpleRemoteEPC() override; Expected<int32_t> runAsMain(ExecutorAddr MainFnAddr, ArrayRef<std::string> Args) override; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h index 85c2d65..2c385de 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h @@ -29,7 +29,7 @@ namespace rt_bootstrap { class LLVM_ABI ExecutorSharedMemoryMapperService final : public ExecutorBootstrapService { public: - ~ExecutorSharedMemoryMapperService(){}; + ~ExecutorSharedMemoryMapperService() override {}; Expected<std::pair<ExecutorAddr, std::string>> reserve(uint64_t Size); Expected<ExecutorAddr> initialize(ExecutorAddr Reservation, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/LibraryResolver.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/LibraryResolver.h new file mode 100644 index 0000000..50d4f6d041 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/LibraryResolver.h @@ -0,0 +1,514 @@ +//===- LibraryResolver.h - Automatic Library Symbol Resolution -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides support for automatically searching symbols across +// dynamic libraries that have not yet been loaded. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_LIBRARYRESOLVER_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_LIBRARYRESOLVER_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ExecutionEngine/Orc/Shared/SymbolFilter.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/LibraryScanner.h" +#include "llvm/Support/Path.h" + +#include <atomic> +#include <shared_mutex> +#include <unordered_map> + +namespace llvm { +namespace orc { + +/// Manages library metadata and state for symbol resolution. +/// +/// Tracks libraries by load state and kind (user/system), and stores +/// associated Bloom filters and hash maps to speed up symbol lookups. +/// Thread-safe for concurrent access. +class LibraryManager { +public: + enum class LibState : uint8_t { Unloaded = 0, Loaded = 1, Queried = 2 }; + + class LibraryInfo { + public: + LibraryInfo(const LibraryInfo &) = delete; + LibraryInfo &operator=(const LibraryInfo &) = delete; + + LibraryInfo(std::string FilePath, LibState S, PathType K, + std::optional<BloomFilter> Filter = std::nullopt) + : FilePath(std::move(FilePath)), S(S), K(K), Filter(std::move(Filter)) { + } + + StringRef getBasePath() const { return sys::path::parent_path(FilePath); } + StringRef getFileName() const { return sys::path::filename(FilePath); } + + std::string getFullPath() const { return FilePath; } + + bool setFilter(BloomFilter F) { + std::lock_guard<std::shared_mutex> Lock(Mtx); + if (Filter) + return false; + Filter.emplace(std::move(F)); + return true; + } + + bool ensureFilterBuilt(const BloomFilterBuilder &FB, + ArrayRef<StringRef> Symbols) { + std::lock_guard<std::shared_mutex> Lock(Mtx); + if (Filter) + return false; + Filter.emplace(FB.build(Symbols)); + return true; + } + + bool mayContain(StringRef Symbol) const { + assert(hasFilter()); + std::shared_lock<std::shared_mutex> Lock(Mtx); + return Filter->mayContain(Symbol); + } + + bool hasFilter() const { + std::shared_lock<std::shared_mutex> Lock(Mtx); + return Filter.has_value(); + } + + LibState getState() const { return S.load(); } + PathType getKind() const { return K; } + + void setState(LibState s) { S.store(s); } + + bool operator==(const LibraryInfo &other) const { + return FilePath == other.FilePath; + } + + private: + std::string FilePath; + std::atomic<LibState> S; + PathType K; + std::optional<BloomFilter> Filter; + mutable std::shared_mutex Mtx; + }; + + /// A read-only view of libraries filtered by state and kind. + /// + /// Lets you loop over only the libraries in a map that match a given State + /// and PathType. + class FilteredView { + public: + using Map = StringMap<std::shared_ptr<LibraryInfo>>; + using Iterator = typename Map::const_iterator; + class FilterIterator { + public: + FilterIterator(Iterator it_, Iterator end_, LibState S, PathType K) + : it(it_), end(end_), S(S), K(K) { + advance(); + } + + bool operator!=(const FilterIterator &other) const { + return it != other.it; + } + + const std::shared_ptr<LibraryInfo> &operator*() const { + return it->second; + } + + FilterIterator &operator++() { + ++it; + advance(); + return *this; + } + + private: + void advance() { + for (; it != end; ++it) + if (it->second->getState() == S && it->second->getKind() == K) + break; + } + Iterator it; + Iterator end; + LibState S; + PathType K; + }; + FilteredView(Iterator begin, Iterator end, LibState s, PathType k) + : mapBegin(begin), mapEnd(end), state(s), kind(k) {} + + FilterIterator begin() const { + return FilterIterator(mapBegin, mapEnd, state, kind); + } + + FilterIterator end() const { + return FilterIterator(mapEnd, mapEnd, state, kind); + } + + private: + Iterator mapBegin; + Iterator mapEnd; + LibState state; + PathType kind; + }; + +private: + StringMap<std::shared_ptr<LibraryInfo>> Libraries; + mutable std::shared_mutex Mtx; + +public: + using LibraryVisitor = std::function<bool(const LibraryInfo &)>; + + LibraryManager() = default; + ~LibraryManager() = default; + + bool addLibrary(std::string Path, PathType Kind, + std::optional<BloomFilter> Filter = std::nullopt) { + std::unique_lock<std::shared_mutex> Lock(Mtx); + if (Libraries.count(Path) > 0) + return false; + Libraries.insert({std::move(Path), + std::make_shared<LibraryInfo>(Path, LibState::Unloaded, + Kind, std::move(Filter))}); + return true; + } + + bool hasLibrary(StringRef Path) const { + std::shared_lock<std::shared_mutex> Lock(Mtx); + if (Libraries.count(Path) > 0) + return true; + return false; + } + + bool removeLibrary(StringRef Path) { + std::unique_lock<std::shared_mutex> Lock(Mtx); + auto I = Libraries.find(Path); + if (I == Libraries.end()) + return false; + Libraries.erase(I); + return true; + } + + void markLoaded(StringRef Path) { + std::unique_lock<std::shared_mutex> Lock(Mtx); + if (auto It = Libraries.find(Path); It != Libraries.end()) + It->second->setState(LibState::Loaded); + } + + void markQueried(StringRef Path) { + std::unique_lock<std::shared_mutex> Lock(Mtx); + if (auto It = Libraries.find(Path); It != Libraries.end()) + It->second->setState(LibState::Queried); + } + + std::shared_ptr<LibraryInfo> getLibrary(StringRef Path) { + std::shared_lock<std::shared_mutex> Lock(Mtx); + if (auto It = Libraries.find(Path); It != Libraries.end()) + return It->second; + return nullptr; + } + + FilteredView getView(LibState S, PathType K) const { + std::shared_lock<std::shared_mutex> Lock(Mtx); + return FilteredView(Libraries.begin(), Libraries.end(), S, K); + } + + void forEachLibrary(const LibraryVisitor &visitor) const { + std::unique_lock<std::shared_mutex> Lock(Mtx); + for (const auto &[_, entry] : Libraries) { + if (!visitor(*entry)) + break; + } + } + + bool isLoaded(StringRef Path) const { + std::unique_lock<std::shared_mutex> Lock(Mtx); + if (auto It = Libraries.find(Path.str()); It != Libraries.end()) + return It->second->getState() == LibState::Loaded; + return false; + } + + bool isQueried(StringRef Path) const { + std::unique_lock<std::shared_mutex> Lock(Mtx); + if (auto It = Libraries.find(Path.str()); It != Libraries.end()) + return It->second->getState() == LibState::Queried; + return false; + } + + void clear() { + std::unique_lock<std::shared_mutex> Lock(Mtx); + Libraries.clear(); + } +}; + +using LibraryInfo = LibraryManager::LibraryInfo; + +struct SearchPlanEntry { + LibraryManager::LibState State; // Loaded, Queried, Unloaded + PathType Type; // User, System +}; + +struct SearchPolicy { + std::vector<SearchPlanEntry> Plan; + + static SearchPolicy defaultPlan() { + return {{{LibraryManager::LibState::Loaded, PathType::User}, + {LibraryManager::LibState::Queried, PathType::User}, + {LibraryManager::LibState::Unloaded, PathType::User}, + {LibraryManager::LibState::Loaded, PathType::System}, + {LibraryManager::LibState::Queried, PathType::System}, + {LibraryManager::LibState::Unloaded, PathType::System}}}; + } +}; + +struct SymbolEnumeratorOptions { + enum Filter : uint32_t { + None = 0, + IgnoreUndefined = 1 << 0, + IgnoreWeak = 1 << 1, + IgnoreIndirect = 1 << 2, + IgnoreHidden = 1 << 3, + IgnoreNonGlobal = 1 << 4 + }; + + static SymbolEnumeratorOptions defaultOptions() { + return {Filter::IgnoreUndefined | Filter::IgnoreWeak | + Filter::IgnoreIndirect}; + } + uint32_t FilterFlags = Filter::None; +}; + +struct SearchConfig { + SearchPolicy Policy; + SymbolEnumeratorOptions Options; + + SearchConfig() + : Policy(SearchPolicy::defaultPlan()), // default plan + Options(SymbolEnumeratorOptions::defaultOptions()) {} +}; + +/// Scans libraries and resolves Symbols across user and system paths. +/// +/// Supports symbol enumeration and filtering via SymbolEnumerator, and tracks +/// symbol resolution results through SymbolQuery. Thread-safe and uses +/// LibraryScanHelper for efficient path resolution and caching. +class LibraryResolver { + friend class LibraryResolutionDriver; + +public: + class SymbolEnumerator { + public: + enum class EnumerateResult { Continue, Stop, Error }; + + using OnEachSymbolFn = std::function<EnumerateResult(StringRef Sym)>; + + static bool enumerateSymbols(StringRef Path, OnEachSymbolFn OnEach, + const SymbolEnumeratorOptions &Opts); + }; + + /// Tracks a set of symbols and the libraries where they are resolved. + /// + /// SymbolQuery is used to keep track of which symbols have been resolved + /// to which libraries. It supports concurrent read/write access using a + /// shared mutex, allowing multiple readers or a single writer at a time. + class SymbolQuery { + public: + /// Holds the result for a single symbol. + struct Result { + std::string Name; + std::string ResolvedLibPath; + }; + + private: + mutable std::shared_mutex Mtx; + StringMap<Result> Results; + std::atomic<size_t> ResolvedCount = 0; + + public: + explicit SymbolQuery(const std::vector<std::string> &Symbols) { + for (const auto &s : Symbols) { + if (!Results.contains(s)) + Results.insert({s, Result{s, ""}}); + } + } + + SmallVector<StringRef> getUnresolvedSymbols() const { + SmallVector<StringRef> Unresolved; + std::shared_lock<std::shared_mutex> Lock(Mtx); + for (const auto &[name, res] : Results) { + if (res.ResolvedLibPath.empty()) + Unresolved.push_back(name); + } + return Unresolved; + } + + void resolve(StringRef Sym, const std::string &LibPath) { + std::unique_lock<std::shared_mutex> Lock(Mtx); + auto It = Results.find(Sym); + if (It != Results.end() && It->second.ResolvedLibPath.empty()) { + It->second.ResolvedLibPath = LibPath; + ResolvedCount.fetch_add(1, std::memory_order_relaxed); + } + } + + bool allResolved() const { + return ResolvedCount.load(std::memory_order_relaxed) == Results.size(); + } + + bool hasUnresolved() const { + return ResolvedCount.load(std::memory_order_relaxed) < Results.size(); + } + + std::optional<StringRef> getResolvedLib(StringRef Sym) const { + std::shared_lock<std::shared_mutex> Lock(Mtx); + auto It = Results.find(Sym); + if (It != Results.end() && !It->second.ResolvedLibPath.empty()) + return StringRef(It->second.ResolvedLibPath); + return std::nullopt; + } + + bool isResolved(StringRef Sym) const { + std::shared_lock<std::shared_mutex> Lock(Mtx); + auto It = Results.find(Sym.str()); + return It != Results.end() && !It->second.ResolvedLibPath.empty(); + } + + std::vector<const Result *> getAllResults() const { + std::shared_lock<std::shared_mutex> Lock(Mtx); + std::vector<const Result *> Out; + Out.reserve(Results.size()); + for (const auto &[_, res] : Results) + Out.push_back(&res); + return Out; + } + }; + + struct Setup { + std::vector<std::string> BasePaths; + std::shared_ptr<LibraryPathCache> Cache; + std::shared_ptr<PathResolver> PResolver; + + size_t ScanBatchSize = 0; + + LibraryScanner::ShouldScanFn ShouldScanCall = [](StringRef) { + return true; + }; + + BloomFilterBuilder FilterBuilder = BloomFilterBuilder(); + + static Setup + create(std::vector<std::string> BasePaths, + std::shared_ptr<LibraryPathCache> existingCache = nullptr, + std::shared_ptr<PathResolver> existingResolver = nullptr, + LibraryScanner::ShouldScanFn customShouldScan = nullptr) { + Setup S; + S.BasePaths = std::move(BasePaths); + + S.Cache = + existingCache ? existingCache : std::make_shared<LibraryPathCache>(); + + S.PResolver = existingResolver ? existingResolver + : std::make_shared<PathResolver>(S.Cache); + + if (customShouldScan) + S.ShouldScanCall = std::move(customShouldScan); + + return S; + } + }; + + LibraryResolver() = delete; + explicit LibraryResolver(const Setup &S); + ~LibraryResolver() = default; + + using OnSearchComplete = unique_function<void(SymbolQuery &)>; + + void dump() { + int i = 0; + LibMgr.forEachLibrary([&](const LibraryInfo &Lib) -> bool { + dbgs() << ++i << ". Library Path : " << Lib.getFullPath() << " -> \n\t\t:" + << " ({Type : (" + << (Lib.getKind() == PathType::User ? "User" : "System") + << ") }, { State : " + << (Lib.getState() == LibraryManager::LibState::Loaded + ? "Loaded" + : "Unloaded") + << "})\n"; + return true; + }); + } + + void searchSymbolsInLibraries(std::vector<std::string> &SymList, + OnSearchComplete OnComplete, + const SearchConfig &Config = SearchConfig()); + +private: + bool scanLibrariesIfNeeded(PathType K, size_t BatchSize = 0); + void resolveSymbolsInLibrary(LibraryInfo &Lib, SymbolQuery &Q, + const SymbolEnumeratorOptions &Opts); + bool + symbolExistsInLibrary(const LibraryInfo &Lib, StringRef Sym, + std::vector<std::string> *MatchedSymbols = nullptr); + + bool symbolExistsInLibrary(const LibraryInfo &Lib, StringRef SymName, + std::vector<std::string> *AllSymbols, + const SymbolEnumeratorOptions &Opts); + + std::shared_ptr<LibraryPathCache> LibPathCache; + std::shared_ptr<PathResolver> LibPathResolver; + LibraryScanHelper ScanHelper; + BloomFilterBuilder FB; + LibraryManager LibMgr; + LibraryScanner::ShouldScanFn ShouldScanCall; + size_t scanBatchSize; +}; + +using SymbolEnumerator = LibraryResolver::SymbolEnumerator; +using SymbolQuery = LibraryResolver::SymbolQuery; +using EnumerateResult = SymbolEnumerator::EnumerateResult; + +class LibraryResolutionDriver { +public: + static std::unique_ptr<LibraryResolutionDriver> + create(const LibraryResolver::Setup &S); + + void addScanPath(const std::string &Path, PathType Kind); + bool markLibraryLoaded(StringRef Path); + bool markLibraryUnLoaded(StringRef Path); + bool isLibraryLoaded(StringRef Path) const { + return LR->LibMgr.isLoaded(Path); + } + + void resetAll() { + LR->LibMgr.clear(); + LR->ScanHelper.resetToScan(); + LR->LibPathCache->clear(); + } + + void scanAll(size_t BatchSize = 0) { + LR->scanLibrariesIfNeeded(PathType::User, BatchSize); + LR->scanLibrariesIfNeeded(PathType::System, BatchSize); + } + + void scan(PathType PK, size_t BatchSize = 0) { + LR->scanLibrariesIfNeeded(PK, BatchSize); + } + + void resolveSymbols(std::vector<std::string> Symbols, + LibraryResolver::OnSearchComplete OnCompletion, + const SearchConfig &Config = SearchConfig()); + + ~LibraryResolutionDriver() = default; + +private: + LibraryResolutionDriver(std::unique_ptr<LibraryResolver> L) + : LR(std::move(L)) {} + + std::unique_ptr<LibraryResolver> LR; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_LIBRARYRESOLVER_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/LibraryScanner.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/LibraryScanner.h new file mode 100644 index 0000000..d1c2013 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/LibraryScanner.h @@ -0,0 +1,474 @@ +//===- LibraryScanner.h - Scanner for Shared Libraries ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides functionality for scanning dynamic (shared) libraries. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_LIBRARYSCANNER_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_LIBRARYSCANNER_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/StringSaver.h" + +#include <atomic> +#include <mutex> +#include <queue> +#include <shared_mutex> +#include <string> +#include <unordered_map> +#include <unordered_set> + +namespace llvm { +namespace orc { + +class LibraryManager; + +class LibraryPathCache { + friend class PathResolver; + +public: + LibraryPathCache() = default; + + void clear(bool isRealPathCache = false) { + std::unique_lock<std::shared_mutex> lock(Mtx); + Seen.clear(); + if (isRealPathCache) { + RealPathCache.clear(); +#ifndef _WIN32 + ReadlinkCache.clear(); + LstatCache.clear(); +#endif + } + } + + void markSeen(const std::string &CanonPath) { + std::unique_lock<std::shared_mutex> lock(Mtx); + Seen.insert(CanonPath); + } + + bool hasSeen(StringRef CanonPath) const { + std::shared_lock<std::shared_mutex> lock(Mtx); + return Seen.contains(CanonPath); + } + + bool hasSeenOrMark(StringRef CanonPath) { + std::string s = CanonPath.str(); + { + std::shared_lock<std::shared_mutex> lock(Mtx); + if (Seen.contains(s)) + return true; + } + { + std::unique_lock<std::shared_mutex> lock(Mtx); + Seen.insert(s); + } + return false; + } + +private: + mutable std::shared_mutex Mtx; + + struct PathInfo { + std::string canonicalPath; + std::error_code ErrnoCode; + }; + + void insert_realpath(StringRef Path, const PathInfo &Info) { + std::unique_lock<std::shared_mutex> lock(Mtx); + RealPathCache.insert({Path, Info}); + } + + std::optional<PathInfo> read_realpath(StringRef Path) const { + std::shared_lock<std::shared_mutex> lock(Mtx); + auto It = RealPathCache.find(Path); + if (It != RealPathCache.end()) + return It->second; + + return std::nullopt; + } + + StringSet<> Seen; + StringMap<PathInfo> RealPathCache; + +#ifndef _WIN32 + StringMap<std::string> ReadlinkCache; + StringMap<mode_t> LstatCache; + + void insert_link(StringRef Path, const std::string &s) { + std::unique_lock<std::shared_mutex> lock(Mtx); + ReadlinkCache.insert({Path, s}); + } + + std::optional<std::string> read_link(StringRef Path) const { + std::shared_lock<std::shared_mutex> lock(Mtx); + auto It = ReadlinkCache.find(Path); + if (It != ReadlinkCache.end()) + return It->second; + + return std::nullopt; + } + + void insert_lstat(StringRef Path, mode_t m) { + std::unique_lock<std::shared_mutex> lock(Mtx); + LstatCache.insert({Path, m}); + } + + std::optional<mode_t> read_lstat(StringRef Path) const { + std::shared_lock<std::shared_mutex> lock(Mtx); + auto It = LstatCache.find(Path); + if (It != LstatCache.end()) + return It->second; + + return std::nullopt; + } + +#endif +}; + +/// Resolves file system paths with optional caching of results. +/// +/// Supports lstat, readlink, and realpath operations. Can resolve paths +/// relative to a base and handle symbolic links. Caches results to reduce +/// repeated system calls when enabled. +class PathResolver { +private: + std::shared_ptr<LibraryPathCache> LibPathCache; + +public: + PathResolver(std::shared_ptr<LibraryPathCache> cache) + : LibPathCache(std::move(cache)) {} + + std::optional<std::string> resolve(StringRef Path, std::error_code &ec) { + return realpathCached(Path, ec); + } +#ifndef _WIN32 + mode_t lstatCached(StringRef Path); + std::optional<std::string> readlinkCached(StringRef Path); +#endif + std::optional<std::string> realpathCached(StringRef Path, std::error_code &ec, + StringRef base = "", + bool baseIsResolved = false, + long symloopLevel = 40); +}; + +/// Performs placeholder substitution in dynamic library paths. +/// +/// Configures known placeholders (like @loader_path) and replaces them +/// in input paths with their resolved values. +class DylibSubstitutor { +public: + void configure(StringRef loaderPath); + + std::string substitute(StringRef input) const { + for (const auto &[ph, value] : Placeholders) { + if (input.starts_with_insensitive(ph)) + return (Twine(value) + input.drop_front(ph.size())).str(); + } + return input.str(); + } + +private: + StringMap<std::string> Placeholders; +}; + +/// Validates and normalizes dynamic library paths. +/// +/// Uses a `PathResolver` to resolve paths to their canonical form and +/// checks whether they point to valid shared libraries. +class DylibPathValidator { +public: + DylibPathValidator(PathResolver &PR) : LibPathResolver(PR) {} + + static bool isSharedLibrary(StringRef Path); + + std::optional<std::string> normalize(StringRef Path) const { + std::error_code ec; + auto real = LibPathResolver.resolve(Path, ec); + if (!real || ec) + return std::nullopt; + + return real; + } + + /// Validate the given path as a shared library. + std::optional<std::string> validate(StringRef Path) const { + auto realOpt = normalize(Path); + if (!realOpt) + return std::nullopt; + + if (!isSharedLibrary(*realOpt)) + return std::nullopt; + + return realOpt; + } + +private: + PathResolver &LibPathResolver; +}; + +enum class SearchPathType { + RPath, + UsrOrSys, + RunPath, +}; + +struct SearchPathConfig { + ArrayRef<StringRef> Paths; + SearchPathType type; +}; + +class SearchPathResolver { +public: + SearchPathResolver(const SearchPathConfig &Cfg, + StringRef PlaceholderPrefix = "") + : Kind(Cfg.type), PlaceholderPrefix(PlaceholderPrefix) { + for (auto &path : Cfg.Paths) + Paths.emplace_back(path.str()); + } + + std::optional<std::string> resolve(StringRef libStem, + const DylibSubstitutor &Subst, + DylibPathValidator &Validator) const; + SearchPathType searchPathType() const { return Kind; } + +private: + std::vector<std::string> Paths; + SearchPathType Kind; + std::string PlaceholderPrefix; +}; + +class DylibResolverImpl { +public: + DylibResolverImpl(DylibSubstitutor Substitutor, DylibPathValidator &Validator, + std::vector<SearchPathResolver> Resolvers) + : Substitutor(std::move(Substitutor)), Validator(Validator), + Resolvers(std::move(Resolvers)) {} + + std::optional<std::string> resolve(StringRef Stem, + bool VariateLibStem = false) const; + +private: + std::optional<std::string> tryWithExtensions(StringRef libstem) const; + + DylibSubstitutor Substitutor; + DylibPathValidator &Validator; + std::vector<SearchPathResolver> Resolvers; +}; + +class DylibResolver { +public: + DylibResolver(DylibPathValidator &Validator) : Validator(Validator) {} + + void configure(StringRef loaderPath, + ArrayRef<SearchPathConfig> SearchPathCfg) { + DylibSubstitutor Substitutor; + Substitutor.configure(loaderPath); + + std::vector<SearchPathResolver> Resolvers; + for (const auto &cfg : SearchPathCfg) { + Resolvers.emplace_back(cfg, + cfg.type == SearchPathType::RPath ? "@rpath" : ""); + } + + impl_ = std::make_unique<DylibResolverImpl>( + std::move(Substitutor), Validator, std::move(Resolvers)); + } + + std::optional<std::string> resolve(StringRef libStem, + bool VariateLibStem = false) const { + if (!impl_) + return std::nullopt; + return impl_->resolve(libStem, VariateLibStem); + } + + static std::string resolvelinkerFlag(StringRef libStem, + StringRef loaderPath) { + DylibSubstitutor Substitutor; + Substitutor.configure(loaderPath); + return Substitutor.substitute(libStem); + } + +private: + DylibPathValidator &Validator; + std::unique_ptr<DylibResolverImpl> impl_; +}; + +enum class PathType : uint8_t { User, System, Unknown }; + +enum class ScanState : uint8_t { NotScanned, Scanning, Scanned }; + +struct LibrarySearchPath { + std::string BasePath; // Canonical base directory path + PathType Kind; // User or System + std::atomic<ScanState> State; + + LibrarySearchPath(std::string Base, PathType K) + : BasePath(std::move(Base)), Kind(K), State(ScanState::NotScanned) {} +}; + +/// Scans and tracks libraries for symbol resolution. +/// +/// Maintains a list of library paths to scan, caches scanned units, +/// and resolves paths canonically for consistent tracking. +class LibraryScanHelper { +public: + explicit LibraryScanHelper(const std::vector<std::string> &SPaths, + std::shared_ptr<LibraryPathCache> LibPathCache, + std::shared_ptr<PathResolver> LibPathResolver) + : LibPathCache(std::move(LibPathCache)), + LibPathResolver(std::move(LibPathResolver)) { + DEBUG_WITH_TYPE( + "orc", dbgs() << "LibraryScanHelper::LibraryScanHelper: base paths : " + << SPaths.size() << "\n";); + for (const auto &p : SPaths) + addBasePath(p); + } + + void + addBasePath(const std::string &P, + PathType Kind = + PathType::Unknown); // Add a canonical directory for scanning + std::vector<std::shared_ptr<LibrarySearchPath>> + getNextBatch(PathType Kind, size_t batchSize); + + bool leftToScan(PathType K) const; + void resetToScan(); + + bool isTrackedBasePath(StringRef P) const; + std::vector<std::shared_ptr<LibrarySearchPath>> getAllUnits() const; + + SmallVector<StringRef> getSearchPaths() const { + SmallVector<StringRef> SearchPaths; + for (const auto &[_, SP] : LibSearchPaths) + SearchPaths.push_back(SP->BasePath); + return SearchPaths; + } + + PathResolver &getPathResolver() const { return *LibPathResolver; } + + LibraryPathCache &getCache() const { return *LibPathCache; } + + bool hasSeenOrMark(StringRef P) const { + return LibPathCache->hasSeenOrMark(P); + } + + std::optional<std::string> resolve(StringRef P, std::error_code &ec) const { + return LibPathResolver->resolve(P.str(), ec); + } + +private: + std::string resolveCanonical(StringRef P, std::error_code &ec) const; + PathType classifyKind(StringRef P) const; + + mutable std::shared_mutex Mtx; + std::shared_ptr<LibraryPathCache> LibPathCache; + std::shared_ptr<PathResolver> LibPathResolver; + + StringMap<std::shared_ptr<LibrarySearchPath>> + LibSearchPaths; // key: canonical path + std::deque<StringRef> UnscannedUsr; + std::deque<StringRef> UnscannedSys; +}; + +/// Loads an object file and provides access to it. +/// +/// Owns the underlying `ObjectFile` and ensures it is valid. +/// Any errors encountered during construction are stored and +/// returned when attempting to access the file. +class ObjectFileLoader { +public: + /// Construct an object file loader from the given path. + explicit ObjectFileLoader(StringRef Path) { + auto ObjOrErr = loadObjectFileWithOwnership(Path); + if (ObjOrErr) + Obj = std::move(*ObjOrErr); + else { + consumeError(std::move(Err)); + Err = ObjOrErr.takeError(); + } + } + + ObjectFileLoader(const ObjectFileLoader &) = delete; + ObjectFileLoader &operator=(const ObjectFileLoader &) = delete; + + ObjectFileLoader(ObjectFileLoader &&) = default; + ObjectFileLoader &operator=(ObjectFileLoader &&) = default; + + /// Get the loaded object file, or return an error if loading failed. + Expected<object::ObjectFile &> getObjectFile() { + if (Err) + return std::move(Err); + return *Obj.getBinary(); + } + + static bool isArchitectureCompatible(const object::ObjectFile &Obj); + +private: + object::OwningBinary<object::ObjectFile> Obj; + Error Err = Error::success(); + + static Expected<object::OwningBinary<object::ObjectFile>> + loadObjectFileWithOwnership(StringRef FilePath); +}; + +/// Scans libraries, resolves dependencies, and registers them. +class LibraryScanner { +public: + using ShouldScanFn = std::function<bool(StringRef)>; + + LibraryScanner( + LibraryScanHelper &H, LibraryManager &LibMgr, + ShouldScanFn ShouldScanCall = [](StringRef path) { return true; }) + : ScanHelper(H), LibMgr(LibMgr), + ShouldScanCall(std::move(ShouldScanCall)) {} + + void scanNext(PathType Kind, size_t batchSize = 1); + + /// Dependency info for a library. + struct LibraryDepsInfo { + llvm::BumpPtrAllocator Alloc; + llvm::StringSaver Saver{Alloc}; + + SmallVector<StringRef, 2> rpath; + SmallVector<StringRef, 2> runPath; + SmallVector<StringRef, 4> deps; + bool isPIE = false; + + void addRPath(StringRef s) { rpath.push_back(Saver.save(s)); } + + void addRunPath(StringRef s) { runPath.push_back(Saver.save(s)); } + + void addDep(StringRef s) { deps.push_back(Saver.save(s)); } + }; + +private: + LibraryScanHelper &ScanHelper; + LibraryManager &LibMgr; + ShouldScanFn ShouldScanCall; + + std::optional<std::string> shouldScan(StringRef FilePath); + Expected<LibraryDepsInfo> extractDeps(StringRef FilePath); + + void handleLibrary(StringRef P, PathType K, int level = 1); + + void scanBaseDir(std::shared_ptr<LibrarySearchPath> U); +}; + +using LibraryDepsInfo = LibraryScanner::LibraryDepsInfo; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_LIBRARYSCANNER_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h index 7526a29d..7ca2ff2 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h @@ -37,7 +37,7 @@ namespace rt_bootstrap { /// Simple page-based allocator. class LLVM_ABI SimpleExecutorDylibManager : public ExecutorBootstrapService { public: - virtual ~SimpleExecutorDylibManager(); + ~SimpleExecutorDylibManager() override; Expected<tpctypes::DylibHandle> open(const std::string &Path, uint64_t Mode); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h index 6224e92..45256ec 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h @@ -32,7 +32,7 @@ namespace rt_bootstrap { /// Simple page-based allocator. class LLVM_ABI SimpleExecutorMemoryManager : public ExecutorBootstrapService { public: - virtual ~SimpleExecutorMemoryManager(); + ~SimpleExecutorMemoryManager() override; Expected<ExecutorAddr> reserve(uint64_t Size); Expected<ExecutorAddr> initialize(tpctypes::FinalizeRequest &FR); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TaskDispatch.h b/llvm/include/llvm/ExecutionEngine/Orc/TaskDispatch.h index 9cf6e00..b73da19 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TaskDispatch.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TaskDispatch.h @@ -37,7 +37,7 @@ class LLVM_ABI Task : public RTTIExtends<Task, RTTIRoot> { public: static char ID; - virtual ~Task() = default; + ~Task() override = default; /// Description of the task to be performed. Used for logging. virtual void printDescription(raw_ostream &OS) = 0; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h index 45834f1..d8506ce 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h @@ -186,9 +186,7 @@ private: SmallVector<ElementId> SortedElems(ContainerElems.begin(), ContainerElems.end()); llvm::sort(SortedElems); - Hash = hash_combine( - Hash, Container, - hash_combine_range(SortedElems.begin(), SortedElems.end())); + Hash = hash_combine(Hash, Container, hash_combine_range(SortedElems)); } return Hash; } diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td index 8e7d9dc..8ce2b1b 100644 --- a/llvm/include/llvm/IR/Attributes.td +++ b/llvm/include/llvm/IR/Attributes.td @@ -410,7 +410,6 @@ def LessPreciseFPMAD : StrBoolAttr<"less-precise-fpmad">; def NoInfsFPMath : StrBoolAttr<"no-infs-fp-math">; def NoNansFPMath : StrBoolAttr<"no-nans-fp-math">; def NoSignedZerosFPMath : StrBoolAttr<"no-signed-zeros-fp-math">; -def UnsafeFPMath : StrBoolAttr<"unsafe-fp-math">; def NoJumpTables : StrBoolAttr<"no-jump-tables">; def NoInlineLineTables : StrBoolAttr<"no-inline-line-tables">; def ProfileSampleAccurate : StrBoolAttr<"profile-sample-accurate">; @@ -474,7 +473,6 @@ def : MergeRule<"setAND<LessPreciseFPMADAttr>">; def : MergeRule<"setAND<NoInfsFPMathAttr>">; def : MergeRule<"setAND<NoNansFPMathAttr>">; def : MergeRule<"setAND<NoSignedZerosFPMathAttr>">; -def : MergeRule<"setAND<UnsafeFPMathAttr>">; def : MergeRule<"setOR<NoImplicitFloatAttr>">; def : MergeRule<"setOR<NoJumpTablesAttr>">; def : MergeRule<"setOR<ProfileSampleAccurateAttr>">; diff --git a/llvm/include/llvm/IR/AutoUpgrade.h b/llvm/include/llvm/IR/AutoUpgrade.h index 31096e8..540d60a 100644 --- a/llvm/include/llvm/IR/AutoUpgrade.h +++ b/llvm/include/llvm/IR/AutoUpgrade.h @@ -96,6 +96,16 @@ namespace llvm { /// info. Return true if module is modified. LLVM_ABI bool UpgradeDebugInfo(Module &M); + /// Copies module attributes to the functions in the module. + /// Currently only effects ARM, Thumb and AArch64 targets. + /// Supported attributes: + /// - branch-target-enforcement + /// - branch-protection-pauth-lr + /// - guarded-control-stack + /// - sign-return-address + /// - sign-return-address-with-bkey + void copyModuleAttrToFunctions(Module &M); + /// Check whether a string looks like an old loop attachment tag. inline bool mayBeOldLoopAttachmentTag(StringRef Name) { return Name.starts_with("llvm.vectorizer."); diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index 12d1c25..4d59ee8 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1487,24 +1487,23 @@ def int_eh_sjlj_setup_dispatch : Intrinsic<[], []>; // def int_var_annotation : DefaultAttrsIntrinsic< [], [llvm_anyptr_ty, llvm_anyptr_ty, LLVMMatchType<1>, llvm_i32_ty, LLVMMatchType<1>], - [IntrInaccessibleMemOnly], "llvm.var.annotation">; + [IntrInaccessibleMemOnly]>; def int_ptr_annotation : DefaultAttrsIntrinsic< [llvm_anyptr_ty], [LLVMMatchType<0>, llvm_anyptr_ty, LLVMMatchType<1>, llvm_i32_ty, LLVMMatchType<1>], - [IntrInaccessibleMemOnly], "llvm.ptr.annotation">; + [IntrInaccessibleMemOnly]>; def int_annotation : DefaultAttrsIntrinsic< [llvm_anyint_ty], [LLVMMatchType<0>, llvm_anyptr_ty, LLVMMatchType<1>, llvm_i32_ty], - [IntrInaccessibleMemOnly], "llvm.annotation">; + [IntrInaccessibleMemOnly]>; // Annotates the current program point with metadata strings which are emitted // as CodeView debug info records. This is expensive, as it disables inlining // and is modelled as having side effects. def int_codeview_annotation : DefaultAttrsIntrinsic<[], [llvm_metadata_ty], - [IntrInaccessibleMemOnly, IntrNoDuplicate], - "llvm.codeview.annotation">; + [IntrInaccessibleMemOnly, IntrNoDuplicate]>; //===------------------------ Trampoline Intrinsics -----------------------===// // @@ -1881,8 +1880,7 @@ def int_clear_cache : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], // Intrinsic to detect whether its argument is a constant. def int_is_constant : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty], - [IntrNoMem, IntrConvergent], - "llvm.is.constant">; + [IntrNoMem, IntrConvergent]>; // Introduce a use of the argument without generating any code. def int_fake_use : DefaultAttrsIntrinsic<[], [llvm_vararg_ty], @@ -2851,7 +2849,15 @@ def int_ptrauth_blend : def int_ptrauth_sign_generic : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; +//===----------------- AllocToken Intrinsics ------------------------------===// + +// Return the token ID for the given !alloc_token metadata. +def int_alloc_token_id : + DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_metadata_ty], + [IntrNoMem, NoUndef<RetIndex>]>; + //===----------------------------------------------------------------------===// + //===------- Convergence Intrinsics ---------------------------------------===// def int_experimental_convergence_entry diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 9e334d4..8e35109 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -3789,6 +3789,20 @@ def int_amdgcn_perm_pk16_b8_u4 : ClangBuiltin<"__builtin_amdgcn_perm_pk16_b8_u4" DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_i64_ty, llvm_i64_ty, llvm_v2i32_ty], [IntrNoMem, IntrSpeculatable]>; +class AMDGPUAddMinMax<LLVMType Ty, string Name> : ClangBuiltin<"__builtin_amdgcn_"#Name>, + DefaultAttrsIntrinsic<[Ty], [Ty, Ty, Ty, llvm_i1_ty /* clamp */], + [IntrNoMem, IntrSpeculatable, ImmArg<ArgIndex<3>>]>; + +def int_amdgcn_add_max_i32 : AMDGPUAddMinMax<llvm_i32_ty, "add_max_i32">; +def int_amdgcn_add_max_u32 : AMDGPUAddMinMax<llvm_i32_ty, "add_max_u32">; +def int_amdgcn_add_min_i32 : AMDGPUAddMinMax<llvm_i32_ty, "add_min_i32">; +def int_amdgcn_add_min_u32 : AMDGPUAddMinMax<llvm_i32_ty, "add_min_u32">; + +def int_amdgcn_pk_add_max_i16 : AMDGPUAddMinMax<llvm_v2i16_ty, "pk_add_max_i16">; +def int_amdgcn_pk_add_max_u16 : AMDGPUAddMinMax<llvm_v2i16_ty, "pk_add_max_u16">; +def int_amdgcn_pk_add_min_i16 : AMDGPUAddMinMax<llvm_v2i16_ty, "pk_add_min_i16">; +def int_amdgcn_pk_add_min_u16 : AMDGPUAddMinMax<llvm_v2i16_ty, "pk_add_min_u16">; + class AMDGPUCooperativeAtomicStore<LLVMType Ty> : Intrinsic < [], [llvm_anyptr_ty, // pointer to store to diff --git a/llvm/include/llvm/IR/IntrinsicsHexagon.td b/llvm/include/llvm/IR/IntrinsicsHexagon.td index 67b873d..20ba51a 100644 --- a/llvm/include/llvm/IR/IntrinsicsHexagon.td +++ b/llvm/include/llvm/IR/IntrinsicsHexagon.td @@ -447,3 +447,15 @@ def int_hexagon_instrprof_custom include "llvm/IR/IntrinsicsHexagonDep.td" + +class Hexagon__ptri32i32v64i16_Intrinsic<string GCCIntSuffix, + list<IntrinsicProperty> intr_properties = [IntrNoMem]> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_ptr_ty,llvm_i32_ty,llvm_i32_ty,llvm_v64i16_ty], + intr_properties>; + +def int_hexagon_V6_vgather_vscattermh : +Hexagon__ptri32i32v64i16_Intrinsic<"HEXAGON_V6_vgather_vscattermh", [IntrArgMemOnly]>; + +def int_hexagon_V6_vgather_vscattermh_128B : +Hexagon__ptri32i32v32i32_Intrinsic<"HEXAGON_V6_vgather_vscattermh_128B", [IntrArgMemOnly]>; diff --git a/llvm/include/llvm/IR/IntrinsicsHexagonDep.td b/llvm/include/llvm/IR/IntrinsicsHexagonDep.td index 0cd6008..fe95377 100644 --- a/llvm/include/llvm/IR/IntrinsicsHexagonDep.td +++ b/llvm/include/llvm/IR/IntrinsicsHexagonDep.td @@ -6832,3 +6832,17 @@ Hexagon_v32i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_hf_f8">; def int_hexagon_V6_vsub_hf_f8_128B : Hexagon_v64i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_hf_f8_128B">; + +// V81 HVX Instructions. + +def int_hexagon_V6_vsub_hf_mix : +Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_hf_mix">; + +def int_hexagon_V6_vsub_hf_mix_128B : +Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_hf_mix_128B">; + +def int_hexagon_V6_vsub_sf_mix : +Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_sf_mix">; + +def int_hexagon_V6_vsub_sf_mix_128B : +Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_sf_mix_128B">; diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td index 3af1750..c9df6c4 100644 --- a/llvm/include/llvm/IR/IntrinsicsNVVM.td +++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td @@ -456,7 +456,7 @@ class WMMA_REGS<string Geom, string Frag, string PtxEltType, bit IsSparse = fals } class WMMA_NAME_LDST<string Op, WMMA_REGS Frag, string Layout, int WithStride> { - string intr = "llvm.nvvm.wmma." + string intr_name = "llvm.nvvm.wmma." # Frag.geom # "." # Op # "." # Frag.frag @@ -467,7 +467,7 @@ class WMMA_NAME_LDST<string Op, WMMA_REGS Frag, string Layout, int WithStride> { // TODO(tra): record name should ideally use the same field order as the intrinsic. // E.g. string record = !subst("llvm", "int", // !subst(".", "_", llvm)); - string record = "int_nvvm_wmma_" + string record_name = "int_nvvm_wmma_" # Frag.geom # "_" # Op # "_" # Frag.frag @@ -496,7 +496,7 @@ class MMA_SIGNATURE<WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> { class WMMA_NAME<string ALayout, string BLayout, int Satfinite, string Rnd, string b1op, WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> { string signature = MMA_SIGNATURE<A, B, C, D>.ret; - string record = "int_nvvm_wmma_" + string record_name = "int_nvvm_wmma_" # A.geom # "_mma" # !subst(".", "_", b1op) @@ -510,7 +510,7 @@ class WMMA_NAME<string ALayout, string BLayout, int Satfinite, string Rnd, strin class MMA_NAME<string ALayout, string BLayout, int Satfinite, string b1op, string Kind, WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> { string signature = MMA_SIGNATURE<A, B, C, D>.ret; - string record = "int_nvvm_mma" + string record_name = "int_nvvm_mma" # !subst(".", "_", b1op) # "_" # A.geom # "_" # ALayout @@ -524,7 +524,7 @@ class MMA_SP_NAME<string Metadata, string Kind, int Satfinite, WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> { string signature = MMA_SIGNATURE<A, B, C, D>.ret; - string record = "int_nvvm_mma" + string record_name = "int_nvvm_mma" # "_" # !subst("::", "_", Metadata) # "_" # A.geom # "_row_col" @@ -533,26 +533,37 @@ class MMA_SP_NAME<string Metadata, string Kind, int Satfinite, # signature; } +// Helper class that takes an intrinsic name and construct a record name. +// Additionally, sets `intr_name` to be non-empty if the default name assigned +// to this intrinsic will not match the name given. +class IntrinsicName<string name> { + string record_name = !subst(".", "_", + !subst("llvm.", "int_", name)); + // Use explicit intrinsic name if it has an _ in it, else rely on LLVM + // assigned default name. + string intr_name = !if(!ne(!find(name, "_"), -1), name, ""); +} + class LDMATRIX_NAME<WMMA_REGS Frag, int Trans> { - string intr = "llvm.nvvm.ldmatrix.sync.aligned" + defvar name = "llvm.nvvm.ldmatrix.sync.aligned" # "." # Frag.geom # "." # Frag.frag # !if(Trans, ".trans", "") # "." # Frag.ptx_elt_type ; - string record = !subst(".", "_", - !subst("llvm.", "int_", intr)); + string intr_name = IntrinsicName<name>.intr_name; + string record_name = IntrinsicName<name>.record_name; } class STMATRIX_NAME<WMMA_REGS Frag, int Trans> { - string intr = "llvm.nvvm.stmatrix.sync.aligned" + defvar name = "llvm.nvvm.stmatrix.sync.aligned" # "." # Frag.geom # "." # Frag.frag # !if(Trans, ".trans", "") # "." # Frag.ptx_elt_type ; - string record = !subst(".", "_", - !subst("llvm.", "int_", intr)); + string intr_name = IntrinsicName<name>.intr_name; + string record_name = IntrinsicName<name>.record_name; } // Generates list of 4-tuples of WMMA_REGS representing a valid MMA op. @@ -1042,45 +1053,49 @@ class NVVM_TCGEN05_MMA_BASE<string Space, bit Sp> { class NVVM_TCGEN05_MMA<bit Sp, string Space, bit AShift, bit ScaleInputD>: NVVM_TCGEN05_MMA_BASE<Space, Sp> { - string intr = "llvm.nvvm.tcgen05.mma" + string name = "llvm.nvvm.tcgen05.mma" # !if(!eq(Sp, 1), ".sp", "") # "." # Space # !if(!eq(ScaleInputD, 1), ".scale_d", "") # !if(!eq(AShift, 1), ".ashift", ""); - string record = !subst(".", "_", !subst("llvm.", "int_", intr)); + string intr_name = IntrinsicName<name>.intr_name; + string record_name = IntrinsicName<name>.record_name; } class NVVM_TCGEN05_MMA_BLOCKSCALE<bit Sp, string Space, string Kind, string ScaleVecSize>: NVVM_TCGEN05_MMA_BASE<Space, Sp> { - string intr = "llvm.nvvm.tcgen05.mma" + string name = "llvm.nvvm.tcgen05.mma" # !if(!eq(Sp, 1), ".sp", "") # "." # Space # "." # Kind # ".block_scale" # ScaleVecSize; - string record = !subst(".", "_", !subst("llvm.", "int_", intr)); + string intr_name = IntrinsicName<name>.intr_name; + string record_name = IntrinsicName<name>.record_name; } class NVVM_TCGEN05_MMA_WS<bit Sp, string Space, bit ZeroColMask>: NVVM_TCGEN05_MMA_BASE<Space, Sp> { - string intr = "llvm.nvvm.tcgen05.mma.ws" + string name = "llvm.nvvm.tcgen05.mma.ws" # !if(!eq(Sp, 1), ".sp", "") # "." # Space # !if(!eq(ZeroColMask, 1), ".zero_col_mask", ""); - string record = !subst(".", "_", !subst("llvm.", "int_", intr)); + string intr_name = IntrinsicName<name>.intr_name; + string record_name = IntrinsicName<name>.record_name; } class NVVM_TCGEN05_MMA_DISABLE_OUTPUT_LANE<bit Sp, string Space, int CtaGroup, bit AShift, bit ScaleInputD>: NVVM_TCGEN05_MMA_BASE<Space, Sp> { - string intr = "llvm.nvvm.tcgen05.mma" + string name = "llvm.nvvm.tcgen05.mma" # !if(!eq(Sp, 1), ".sp", "") # "." # Space # !if(!eq(ScaleInputD, 1), ".scale_d", "") # ".disable_output_lane.cg" # CtaGroup # !if(!eq(AShift, 1), ".ashift", ""); - string record = !subst(".", "_", !subst("llvm.", "int_", intr)); + string intr_name = IntrinsicName<name>.intr_name; + string record_name = IntrinsicName<name>.record_name; } class NVVM_TCGEN05_MMA_BLOCKSCALE_SUPPORTED<string Kind, string ScaleVecSize> { @@ -2273,7 +2288,7 @@ class NVVM_WMMA_LD<WMMA_REGS Frag, string Layout, int WithStride> : Intrinsic<Frag.regs, !if(WithStride, [llvm_anyptr_ty, llvm_i32_ty], [llvm_anyptr_ty]), [IntrWillReturn, IntrReadMem, IntrArgMemOnly, IntrNoCallback, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], - WMMA_NAME_LDST<"load", Frag, Layout, WithStride>.intr>; + WMMA_NAME_LDST<"load", Frag, Layout, WithStride>.intr_name>; // WMMA.STORE.D class NVVM_WMMA_ST<WMMA_REGS Frag, string Layout, int WithStride> @@ -2283,18 +2298,18 @@ class NVVM_WMMA_ST<WMMA_REGS Frag, string Layout, int WithStride> Frag.regs, !if(WithStride, [llvm_i32_ty], [])), [IntrWriteMem, IntrArgMemOnly, IntrNoCallback, WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], - WMMA_NAME_LDST<"store", Frag, Layout, WithStride>.intr>; + WMMA_NAME_LDST<"store", Frag, Layout, WithStride>.intr_name>; // Create all load/store variants foreach layout = ["row", "col"] in { foreach stride = [0, 1] in { foreach frag = NVVM_MMA_OPS.all_ld_ops in if NVVM_WMMA_LDST_SUPPORTED<frag, layout>.ret then - def WMMA_NAME_LDST<"load", frag, layout, stride>.record + def WMMA_NAME_LDST<"load", frag, layout, stride>.record_name : NVVM_WMMA_LD<frag, layout, stride>; foreach frag = NVVM_MMA_OPS.all_st_ops in if NVVM_WMMA_LDST_SUPPORTED<frag, layout>.ret then - def WMMA_NAME_LDST<"store", frag, layout, stride>.record + def WMMA_NAME_LDST<"store", frag, layout, stride>.record_name : NVVM_WMMA_ST<frag, layout, stride>; } } @@ -2313,7 +2328,7 @@ foreach layout_a = ["row", "col"] in { foreach b1op = NVVM_MMA_B1OPS<op>.ret in { if NVVM_WMMA_SUPPORTED<op, layout_a, layout_b, satf, rnd>.ret then { def WMMA_NAME<layout_a, layout_b, satf, rnd, b1op, - op[0], op[1], op[2], op[3]>.record + op[0], op[1], op[2], op[3]>.record_name : NVVM_MMA<op[0], op[1], op[2], op[3]>; } } // b1op @@ -2330,7 +2345,7 @@ foreach layout_a = ["row", "col"] in { foreach b1op = NVVM_MMA_B1OPS<op>.ret in { foreach kind = ["", "kind::f8f6f4"] in { if NVVM_MMA_SUPPORTED<op, layout_a, layout_b, kind, satf>.ret then { - def MMA_NAME<layout_a, layout_b, satf, b1op, kind, op[0], op[1], op[2], op[3]>.record + def MMA_NAME<layout_a, layout_b, satf, b1op, kind, op[0], op[1], op[2], op[3]>.record_name : NVVM_MMA<op[0], op[1], op[2], op[3]>; } } // kind @@ -2379,7 +2394,7 @@ foreach metadata = ["sp", "sp::ordered_metadata"] in { foreach op = NVVM_MMA_OPS.all_mma_sp_ops in { if NVVM_MMA_SP_SUPPORTED<op, metadata, kind, satf>.ret then { def MMA_SP_NAME<metadata, kind, satf, - op[0], op[1], op[2], op[3]>.record + op[0], op[1], op[2], op[3]>.record_name : NVVM_MMA_SP<op[0], op[1], op[2], op[3]>; } } // op @@ -2392,12 +2407,12 @@ class NVVM_LDMATRIX<WMMA_REGS Frag, int Transposed> : Intrinsic<Frag.regs, [llvm_anyptr_ty], [IntrReadMem, IntrArgMemOnly, IntrNoCallback, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], - LDMATRIX_NAME<Frag, Transposed>.intr>; + LDMATRIX_NAME<Frag, Transposed>.intr_name>; foreach transposed = [0, 1] in { foreach frag = NVVM_MMA_OPS.all_ldmatrix_ops in { if NVVM_LDMATRIX_SUPPORTED<frag, transposed>.ret then { - def LDMATRIX_NAME<frag, transposed>.record + def LDMATRIX_NAME<frag, transposed>.record_name : NVVM_LDMATRIX<frag, transposed>; } } @@ -2409,12 +2424,12 @@ class NVVM_STMATRIX<WMMA_REGS Frag, int Transposed> !listconcat([llvm_anyptr_ty], Frag.regs), [IntrWriteMem, IntrArgMemOnly, IntrNoCallback, WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>], - STMATRIX_NAME<Frag, Transposed>.intr>; + STMATRIX_NAME<Frag, Transposed>.intr_name>; foreach transposed = [0, 1] in { foreach frag = NVVM_MMA_OPS.all_stmatrix_ops in { if NVVM_STMATRIX_SUPPORTED<frag, transposed>.ret then { - def STMATRIX_NAME<frag, transposed>.record + def STMATRIX_NAME<frag, transposed>.record_name : NVVM_STMATRIX<frag, transposed>; } } @@ -2767,14 +2782,15 @@ foreach cta_group = ["cg1", "cg2"] in { "64x128b_warpx2_02_13", "64x128b_warpx2_01_23", "32x128b_warpx4"] in { - defvar intr_suffix = StrJoin<"_", [shape, src_fmt, cta_group]>.ret; - defvar name_suffix = StrJoin<".", [shape, src_fmt, cta_group]>.ret; + defvar name = "llvm.nvvm.tcgen05.cp." # + StrJoin<".", [shape, src_fmt, cta_group]>.ret; - def int_nvvm_tcgen05_cp_ # intr_suffix : Intrinsic<[], + defvar intrinsic_name = IntrinsicName<name>; + def intrinsic_name.record_name : Intrinsic<[], [llvm_tmem_ptr_ty, // tmem_addr llvm_i64_ty], // smem descriptor [IntrConvergent, IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<0>>], - "llvm.nvvm.tcgen05.cp." # name_suffix>; + intrinsic_name.intr_name>; } } } @@ -2881,9 +2897,9 @@ foreach sp = [0, 1] in { ] ); - def mma.record: + def mma.record_name: DefaultAttrsIntrinsicFlags<[], args, flags, intrinsic_properties, - mma.intr>; + mma.intr_name>; } } } @@ -2918,8 +2934,8 @@ foreach sp = [0, 1] in { Range<ArgIndex<!add(nargs, 1)>, 0, !if(!eq(ashift, 1), 2, 4)>] ); - def mma.record: DefaultAttrsIntrinsicFlags<[], args, flags, intrinsic_properties, - mma.intr>; + def mma.record_name : DefaultAttrsIntrinsicFlags<[], args, flags, + intrinsic_properties, mma.intr_name>; } // ashift } // scale_d } // cta_group @@ -2944,11 +2960,11 @@ foreach sp = [0, 1] in { defvar collector_usage = ArgIndex<!add(nargs, 1)>; if NVVM_TCGEN05_MMA_BLOCKSCALE_SUPPORTED<kind, scale_vec_size>.ret then { - def mma.record: DefaultAttrsIntrinsicFlags<[], args, flags, + def mma.record_name : DefaultAttrsIntrinsicFlags<[], args, flags, !listconcat(mma.common_intr_props, [Range<cta_group, 1, 3>, Range<collector_usage, 0, 4>]), - mma.intr>; + mma.intr_name>; } } } @@ -2977,9 +2993,9 @@ foreach sp = [0, 1] in { Range<ArgIndex<!add(nargs, 2)>, 0, 4>] ); - def mma.record: + def mma.record_name: DefaultAttrsIntrinsicFlags<[], args, flags, intrinsic_properties, - mma.intr>; + mma.intr_name>; } } } diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td b/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td index 4a0272c..b1bb24e 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td @@ -162,6 +162,10 @@ let TargetPrefix = "riscv" in { defm "" : RISCVSFCustomVC_XVV<["x", "i", "v", "f"]>; defm "" : RISCVSFCustomVC_XVW<["x", "i", "v", "f"]>; + // XSfvfexp* and XSfvfexpa* + defm sf_vfexp : RISCVUnaryAA; + defm sf_vfexpa : RISCVUnaryAA; + // XSfvqmaccdod def int_riscv_sf_vqmaccu_2x8x2 : RISCVSFCustomVMACC; def int_riscv_sf_vqmacc_2x8x2 : RISCVSFCustomVMACC; diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h index 0062cec..eb60bee 100644 --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -172,9 +172,13 @@ struct alignas(8) GlobalValueSummaryInfo { /// Add a summary corresponding to a global value definition in a module with /// the corresponding GUID. - void addSummary(std::unique_ptr<GlobalValueSummary> Summary) { - return SummaryList.push_back(std::move(Summary)); - } + inline void addSummary(std::unique_ptr<GlobalValueSummary> Summary); + + /// Verify that the HasLocal flag is consistent with the SummaryList. Should + /// only be called prior to index-based internalization and promotion. + inline void verifyLocal() const; + + bool hasLocal() const { return HasLocal; } private: /// List of global value summary structures for a particular value held @@ -183,6 +187,22 @@ private: /// compiling without sufficient distinguishing path, or (theoretically) hash /// collisions. Each summary is from a different module. GlobalValueSummaryList SummaryList; + + /// True if the SummaryList contains at least one summary with local linkage. + /// In most cases there should be only one, unless translation units with + /// same-named locals were compiled without distinguishing path. And generally + /// there should not be a mix of local and non-local summaries, because the + /// GUID for a local is computed with the path prepended and a ';' delimiter. + /// In extremely rare cases there could be a GUID hash collision. Having the + /// flag saves having to walk through all summaries to prove the existence or + /// not of any locals. + /// NOTE: this flag is set when the index is built. It does not reflect + /// index-based internalization and promotion decisions. Generally most + /// index-based analysis occurs before then, but any users should assert that + /// the withInternalizeAndPromote() flag is not set on the index. + /// TODO: Replace checks in various ThinLTO analyses that loop through all + /// summaries to handle the local case with a check of the flag. + bool HasLocal : 1; }; /// Map from global value GUID to corresponding summary structures. Use a @@ -219,6 +239,10 @@ struct ValueInfo { return getRef()->second.getSummaryList(); } + void verifyLocal() const { getRef()->second.verifyLocal(); } + + bool hasLocal() const { return getRef()->second.hasLocal(); } + // Even if the index is built with GVs available, we may not have one for // summary entries synthesized for profiled indirect call targets. bool hasName() const { return !haveGVs() || getValue(); } @@ -649,7 +673,23 @@ public: friend class ModuleSummaryIndex; }; -GlobalValueSummaryInfo::GlobalValueSummaryInfo(bool HaveGVs) : U(HaveGVs) {} +GlobalValueSummaryInfo::GlobalValueSummaryInfo(bool HaveGVs) + : U(HaveGVs), HasLocal(false) {} + +void GlobalValueSummaryInfo::addSummary( + std::unique_ptr<GlobalValueSummary> Summary) { + if (GlobalValue::isLocalLinkage(Summary->linkage())) + HasLocal = true; + return SummaryList.push_back(std::move(Summary)); +} + +void GlobalValueSummaryInfo::verifyLocal() const { + assert(HasLocal == + llvm::any_of(SummaryList, + [](const std::unique_ptr<GlobalValueSummary> &Summary) { + return GlobalValue::isLocalLinkage(Summary->linkage()); + })); +} /// Alias summary information. class AliasSummary : public GlobalValueSummary { @@ -1449,6 +1489,9 @@ private: /// every summary of a GV is synchronized. bool WithDSOLocalPropagation = false; + /// Indicates that summary-based internalization and promotion has run. + bool WithInternalizeAndPromote = false; + /// Indicates that we have whole program visibility. bool WithWholeProgramVisibility = false; @@ -1653,6 +1696,9 @@ public: bool withDSOLocalPropagation() const { return WithDSOLocalPropagation; } void setWithDSOLocalPropagation() { WithDSOLocalPropagation = true; } + bool withInternalizeAndPromote() const { return WithInternalizeAndPromote; } + void setWithInternalizeAndPromote() { WithInternalizeAndPromote = true; } + bool withWholeProgramVisibility() const { return WithWholeProgramVisibility; } void setWithWholeProgramVisibility() { WithWholeProgramVisibility = true; } diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 99f70b1..e3ec7e1 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -198,6 +198,26 @@ struct constantexpr_match { /// expression. inline constantexpr_match m_ConstantExpr() { return constantexpr_match(); } +template <typename SubPattern_t> struct Splat_match { + SubPattern_t SubPattern; + Splat_match(const SubPattern_t &SP) : SubPattern(SP) {} + + template <typename OpTy> bool match(OpTy *V) const { + if (auto *C = dyn_cast<Constant>(V)) { + auto *Splat = C->getSplatValue(); + return Splat ? SubPattern.match(Splat) : false; + } + // TODO: Extend to other cases (e.g. shufflevectors). + return false; + } +}; + +/// Match a constant splat. TODO: Extend this to non-constant splats. +template <typename T> +inline Splat_match<T> m_ConstantSplat(const T &SubPattern) { + return SubPattern; +} + /// Match an arbitrary basic block value and ignore it. inline class_match<BasicBlock> m_BasicBlock() { return class_match<BasicBlock>(); @@ -2925,6 +2945,12 @@ inline typename m_Intrinsic_Ty<Opnd0>::Ty m_VecReverse(const Opnd0 &Op0) { return m_Intrinsic<Intrinsic::vector_reverse>(Op0); } +template <typename Opnd0, typename Opnd1, typename Opnd2> +inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty +m_VectorInsert(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { + return m_Intrinsic<Intrinsic::vector_insert>(Op0, Op1, Op2); +} + //===----------------------------------------------------------------------===// // Matchers for two-operands operators with the operators in either order // diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.h b/llvm/include/llvm/IR/RuntimeLibcalls.h index ada3523..0135989 100644 --- a/llvm/include/llvm/IR/RuntimeLibcalls.h +++ b/llvm/include/llvm/IR/RuntimeLibcalls.h @@ -236,7 +236,7 @@ private: static bool hasAEABILibcalls(const Triple &TT) { return TT.isTargetAEABI() || TT.isTargetGNUAEABI() || - TT.isTargetMuslAEABI() || TT.isAndroid(); + TT.isTargetMuslAEABI() || TT.isOSFuchsia() || TT.isAndroid(); } LLVM_READONLY diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.td b/llvm/include/llvm/IR/RuntimeLibcalls.td index a8b647c..ec16995 100644 --- a/llvm/include/llvm/IR/RuntimeLibcalls.td +++ b/llvm/include/llvm/IR/RuntimeLibcalls.td @@ -32,6 +32,15 @@ def isWindowsMSVCEnvironment : RuntimeLibcallPredicate< def isNotOSLinuxAndNotOSOpenBSD : RuntimeLibcallPredicate< [{!TT.isOSLinux() && !TT.isOSOpenBSD()}]>; +def isNotOSAIXAndNotOSOpenBSD : RuntimeLibcallPredicate< + [{!TT.isOSAIX() && !TT.isOSOpenBSD()}]>; + +// OpenBSD uses __guard_local. AIX uses __ssp_canary_word, MSVC/Windows +// Itanium uses __security_cookie +def hasStackChkFail : RuntimeLibcallPredicate< + [{ !TT.isOSOpenBSD() && !TT.isWindowsMSVCEnvironment() && + !TT.isWindowsItaniumEnvironment()}]>; + def isWindowsMSVCOrItaniumEnvironment : RuntimeLibcallPredicate< [{TT.isWindowsMSVCEnvironment() || TT.isWindowsItaniumEnvironment()}]>; @@ -1241,9 +1250,9 @@ defvar LibmHasLdexpF80 = LibcallImpls<(add ldexpl_f80), isNotOSWindowsOrIsCygwin defvar LibmHasFrexpF128 = LibcallImpls<(add frexpl_f128), isNotOSWindowsOrIsCygwinMinGW>; defvar LibmHasLdexpF128 = LibcallImpls<(add ldexpl_f128), isNotOSWindowsOrIsCygwinMinGW>; -defvar has__stack_chk_fail = LibcallImpls<(add __stack_chk_fail), isNotOSOpenBSD>; +defvar has__stack_chk_fail = LibcallImpls<(add __stack_chk_fail), hasStackChkFail>; defvar has__stack_chk_guard = - LibcallImpls<(add __stack_chk_guard), isNotOSOpenBSD>; + LibcallImpls<(add __stack_chk_guard), hasStackChkFail>; defvar has__stack_smash_handler = LibcallImpls<(add __stack_smash_handler), isOSOpenBSD>; defvar has___guard_local = LibcallImpls<(add __guard_local), isOSOpenBSD>; @@ -1396,8 +1405,8 @@ defvar ExceptionModelCallsArm64EC = (add def WindowsARM64ECSystemLibrary : SystemRuntimeLibrary<isWindowsArm64EC, (add WinArm64ECDefaultRuntimeLibcallImpls, - arm64ec___stack_chk_fail, - __stack_chk_guard, + AvailableIf<arm64ec___stack_chk_fail, hasStackChkFail>, + AvailableIf<__stack_chk_guard, hasStackChkFail>, LibcallImpls<(add __security_check_cookie_arm64ec, __security_cookie), isWindowsMSVCEnvironment>, @@ -2117,7 +2126,9 @@ defvar MSP430DefaultOptOut = [ __fixdfdi, __fixunsdfsi, __modsi3, __floatunsisf, __fixunsdfdi, __ltsf2, __floatdisf, __floatdidf, __lshrsi3, __subsf3, __umodhi3, __floatunsidf, - __floatundidf + __floatundidf, __gtdf2, __eqdf2, __gedf2, __ltdf2, __ledf2, + __adddf3, __divdf3, __divdi3, __moddi3, + __muldf3, __subdf3, __udivdi3, __umoddi3 ]; // EABI Libcalls - EABI Section 6.2 @@ -2316,11 +2327,11 @@ def PPCSystemLibrary LibmHasSinCosPPCF128, AvailableIf<memcpy, isNotAIX>, LibcallImpls<(add Int128RTLibcalls), isPPC64>, - has__stack_chk_fail, has__stack_smash_handler, has___guard_local, AvailableIf<__ssp_canary_word, isAIX>, - AvailableIf<__stack_chk_guard, isNotAIX>)>; + AvailableIf<__stack_chk_fail, isNotOSOpenBSD>, + AvailableIf<__stack_chk_guard, isNotOSAIXAndNotOSOpenBSD>)>; //===----------------------------------------------------------------------===// // RISCV Runtime Libcalls diff --git a/llvm/include/llvm/IRReader/IRReader.h b/llvm/include/llvm/IRReader/IRReader.h index 790140f..00cf12d 100644 --- a/llvm/include/llvm/IRReader/IRReader.h +++ b/llvm/include/llvm/IRReader/IRReader.h @@ -15,6 +15,7 @@ #define LLVM_IRREADER_IRREADER_H #include "llvm/ADT/StringRef.h" +#include "llvm/AsmParser/AsmParserContext.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Support/Compiler.h" #include <memory> @@ -50,19 +51,19 @@ getLazyIRFileModule(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context, /// for it. Otherwise, attempt to parse it as LLVM Assembly and return /// a Module for it. /// \param DataLayoutCallback Override datalayout in the llvm assembly. -LLVM_ABI std::unique_ptr<Module> parseIR(MemoryBufferRef Buffer, - SMDiagnostic &Err, - LLVMContext &Context, - ParserCallbacks Callbacks = {}); +LLVM_ABI std::unique_ptr<Module> +parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err, LLVMContext &Context, + ParserCallbacks Callbacks = {}, + AsmParserContext *ParserContext = nullptr); /// If the given file holds a bitcode image, return a Module for it. /// Otherwise, attempt to parse it as LLVM Assembly and return a Module /// for it. /// \param DataLayoutCallback Override datalayout in the llvm assembly. -LLVM_ABI std::unique_ptr<Module> parseIRFile(StringRef Filename, - SMDiagnostic &Err, - LLVMContext &Context, - ParserCallbacks Callbacks = {}); +LLVM_ABI std::unique_ptr<Module> +parseIRFile(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context, + ParserCallbacks Callbacks = {}, + AsmParserContext *ParserContext = nullptr); } #endif diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index cd774e7..581b4ad 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -62,6 +62,7 @@ LLVM_ABI void initializeBasicBlockSectionsPass(PassRegistry &); LLVM_ABI void initializeBarrierNoopPass(PassRegistry &); LLVM_ABI void initializeBasicAAWrapperPassPass(PassRegistry &); LLVM_ABI void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry &); +LLVM_ABI void initializeMachineBlockHashInfoPass(PassRegistry &); LLVM_ABI void initializeBranchFolderLegacyPass(PassRegistry &); LLVM_ABI void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry &); LLVM_ABI void initializeBranchRelaxationLegacyPass(PassRegistry &); @@ -222,6 +223,7 @@ LLVM_ABI void initializeMachineSanitizerBinaryMetadataLegacyPass(PassRegistry &); LLVM_ABI void initializeMIR2VecVocabLegacyAnalysisPass(PassRegistry &); LLVM_ABI void initializeMIR2VecVocabPrinterLegacyPassPass(PassRegistry &); +LLVM_ABI void initializeMIR2VecPrinterLegacyPassPass(PassRegistry &); LLVM_ABI void initializeMachineSchedulerLegacyPass(PassRegistry &); LLVM_ABI void initializeMachineSinkingLegacyPass(PassRegistry &); LLVM_ABI void initializeMachineTraceMetricsWrapperPassPass(PassRegistry &); diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 000472f..3a4dc5a 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -317,6 +317,8 @@ LLVM_ABI ThinBackend createInProcessThinBackend( /// distributor. /// RemoteCompiler specifies the path to a Clang executable to be invoked for /// the backend jobs. +/// RemoteCompilerPrependArgs specifies a list of prepend arguments to be +/// applied to the backend compilations. /// RemoteCompilerArgs specifies a list of arguments to be applied to the /// backend compilations. /// SaveTemps is a debugging tool that prevents temporary files created by this @@ -326,6 +328,7 @@ LLVM_ABI ThinBackend createOutOfProcessThinBackend( bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, StringRef LinkerOutputFile, StringRef Distributor, ArrayRef<StringRef> DistributorArgs, StringRef RemoteCompiler, + ArrayRef<StringRef> RemoteCompilerPrependArgs, ArrayRef<StringRef> RemoteCompilerArgs, bool SaveTemps); /// This ThinBackend writes individual module indexes to files, instead of @@ -459,6 +462,19 @@ private: ModuleMapType ModuleMap; // The bitcode modules to compile, if specified by the LTO Config. std::optional<ModuleMapType> ModulesToCompile; + + void setPrevailingModuleForGUID(GlobalValue::GUID GUID, StringRef Module) { + PrevailingModuleForGUID[GUID] = Module; + } + bool isPrevailingModuleForGUID(GlobalValue::GUID GUID, + StringRef Module) const { + auto It = PrevailingModuleForGUID.find(GUID); + return It != PrevailingModuleForGUID.end() && It->second == Module; + } + + private: + // Make this private so all accesses must go through above accessor methods + // to avoid inadvertently creating new entries on lookups. DenseMap<GlobalValue::GUID, StringRef> PrevailingModuleForGUID; } ThinLTO; diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h index 4a528ee..74abe34 100644 --- a/llvm/include/llvm/MC/MCContext.h +++ b/llvm/include/llvm/MC/MCContext.h @@ -175,7 +175,7 @@ private: unsigned GetInstance(unsigned LocalLabelVal); /// SHT_LLVM_BB_ADDR_MAP version to emit. - uint8_t BBAddrMapVersion = 3; + uint8_t BBAddrMapVersion = 4; /// The file name of the log file from the environment variable /// AS_SECURE_LOG_FILE. Which must be set before the .secure_log_unique diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index 6a235c0..2f207925 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -736,8 +736,8 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addISelPasses( addPass(LowerEmuTLSPass()); addPass(PreISelIntrinsicLoweringPass(&TM)); - addPass(ExpandLargeDivRemPass(&TM)); - addPass(ExpandFpPass(&TM, getOptLevel())); + addPass(ExpandLargeDivRemPass(TM)); + addPass(ExpandFpPass(TM, getOptLevel())); derived().addIRPasses(addPass); derived().addCodeGenPrepare(addPass); @@ -773,7 +773,7 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addIRPasses( // target lowering hook. if (!Opt.DisableMergeICmps) addPass(MergeICmpsPass()); - addPass(ExpandMemCmpPass(&TM)); + addPass(ExpandMemCmpPass(TM)); } // Run GC lowering passes for builtin collectors @@ -812,7 +812,7 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addIRPasses( // Convert conditional moves to conditional jumps when profitable. if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize) - addPass(SelectOptimizePass(&TM)); + addPass(SelectOptimizePass(TM)); if (Opt.EnableGlobalMergeFunc) addPass(GlobalMergeFuncPass()); @@ -839,14 +839,14 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addPassesToHandleExceptions( case ExceptionHandling::ARM: case ExceptionHandling::AIX: case ExceptionHandling::ZOS: - addPass(DwarfEHPreparePass(&TM)); + addPass(DwarfEHPreparePass(TM)); break; case ExceptionHandling::WinEH: // We support using both GCC-style and MSVC-style exceptions on Windows, so // add both preparation passes. Each pass will only actually run if it // recognizes the personality function. addPass(WinEHPreparePass()); - addPass(DwarfEHPreparePass(&TM)); + addPass(DwarfEHPreparePass(TM)); break; case ExceptionHandling::Wasm: // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs @@ -871,7 +871,7 @@ template <typename Derived, typename TargetMachineT> void CodeGenPassBuilder<Derived, TargetMachineT>::addCodeGenPrepare( AddIRPass &addPass) const { if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP) - addPass(CodeGenPreparePass(&TM)); + addPass(CodeGenPreparePass(TM)); // TODO: Default ctor'd RewriteSymbolPass is no-op. // addPass(RewriteSymbolPass()); } @@ -892,8 +892,8 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addISelPrepare( addPass(CallBrPreparePass()); // Add both the safe stack and the stack protection passes: each of them will // only protect functions that have corresponding attributes. - addPass(SafeStackPass(&TM)); - addPass(StackProtectorPass(&TM)); + addPass(SafeStackPass(TM)); + addPass(StackProtectorPass(TM)); if (Opt.PrintISelInput) addPass(PrintFunctionPass(dbgs(), diff --git a/llvm/include/llvm/Support/AllocToken.h b/llvm/include/llvm/Support/AllocToken.h new file mode 100644 index 0000000..e40d816 --- /dev/null +++ b/llvm/include/llvm/Support/AllocToken.h @@ -0,0 +1,68 @@ +//===- llvm/Support/AllocToken.h - Allocation Token Calculation -----*- C++ -*// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Definition of AllocToken modes and shared calculation of stateless token IDs. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_ALLOCTOKEN_H +#define LLVM_SUPPORT_ALLOCTOKEN_H + +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include <cstdint> +#include <optional> + +namespace llvm { + +/// Modes for generating allocation token IDs. +enum class AllocTokenMode { + /// Incrementally increasing token ID. + Increment, + + /// Simple mode that returns a statically-assigned random token ID. + Random, + + /// Token ID based on allocated type hash. + TypeHash, + + /// Token ID based on allocated type hash, where the top half ID-space is + /// reserved for types that contain pointers and the bottom half for types + /// that do not contain pointers. + TypeHashPointerSplit, +}; + +/// The default allocation token mode. +inline constexpr AllocTokenMode DefaultAllocTokenMode = + AllocTokenMode::TypeHashPointerSplit; + +/// Returns the AllocTokenMode from its canonical string name; if an invalid +/// name was provided returns nullopt. +LLVM_ABI std::optional<AllocTokenMode> +getAllocTokenModeFromString(StringRef Name); + +/// Metadata about an allocation used to generate a token ID. +struct AllocTokenMetadata { + SmallString<64> TypeName; + bool ContainsPointer; +}; + +/// Calculates stable allocation token ID. Returns std::nullopt for stateful +/// modes that are only available in the AllocToken pass. +/// +/// \param Mode The token generation mode. +/// \param Metadata The metadata about the allocation. +/// \param MaxTokens The maximum number of tokens (must not be 0) +/// \return The calculated allocation token ID, or std::nullopt. +LLVM_ABI std::optional<uint64_t> +getAllocToken(AllocTokenMode Mode, const AllocTokenMetadata &Metadata, + uint64_t MaxTokens); + +} // end namespace llvm + +#endif // LLVM_SUPPORT_ALLOCTOKEN_H diff --git a/llvm/include/llvm/Support/GlobPattern.h b/llvm/include/llvm/Support/GlobPattern.h index c1b4484..8cae6a3 100644 --- a/llvm/include/llvm/Support/GlobPattern.h +++ b/llvm/include/llvm/Support/GlobPattern.h @@ -63,21 +63,30 @@ public: // Returns true for glob pattern "*". Can be used to avoid expensive // preparation/acquisition of the input for match(). bool isTrivialMatchAll() const { - if (!Prefix.empty()) + if (PrefixSize) return false; - if (!Suffix.empty()) + if (SuffixSize) return false; if (SubGlobs.size() != 1) return false; return SubGlobs[0].getPat() == "*"; } - StringRef prefix() const { return Prefix; } - StringRef suffix() const { return Suffix; } + // The following functions are just shortcuts for faster matching. They are + // conservative to simplify implementations. + + // Returns plain prefix of the pattern. + StringRef prefix() const { return Pattern.take_front(PrefixSize); } + // Returns plain suffix of the pattern. + StringRef suffix() const { return Pattern.take_back(SuffixSize); } + // Returns the longest plain substring of the pattern between prefix and + // suffix. + StringRef longest_substr() const; private: - StringRef Prefix; - StringRef Suffix; + StringRef Pattern; + size_t PrefixSize = 0; + size_t SuffixSize = 0; struct SubGlobPattern { /// \param Pat the pattern to match against diff --git a/llvm/include/llvm/Support/Timer.h b/llvm/include/llvm/Support/Timer.h index 6a44758..527d67f 100644 --- a/llvm/include/llvm/Support/Timer.h +++ b/llvm/include/llvm/Support/Timer.h @@ -209,6 +209,7 @@ class TimerGroup { std::string Description; Timer *FirstTimer = nullptr; ///< First timer in the group. std::vector<PrintRecord> TimersToPrint; + bool PrintOnExit; TimerGroup **Prev; ///< Pointer to Next field of previous timergroup in list. TimerGroup *Next; ///< Pointer to next timergroup in list. @@ -217,13 +218,15 @@ class TimerGroup { friend class TimerGlobals; explicit TimerGroup(StringRef Name, StringRef Description, - sys::SmartMutex<true> &lock); + sys::SmartMutex<true> &lock, bool PrintOnExit); public: - LLVM_ABI explicit TimerGroup(StringRef Name, StringRef Description); + LLVM_ABI explicit TimerGroup(StringRef Name, StringRef Description, + bool PrintOnExit = true); LLVM_ABI explicit TimerGroup(StringRef Name, StringRef Description, - const StringMap<TimeRecord> &Records); + const StringMap<TimeRecord> &Records, + bool PrintOnExit = true); LLVM_ABI ~TimerGroup(); diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index 3d21f52..47d5d68 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -484,13 +484,6 @@ def propagate_undef_shuffle_mask: GICombineRule< [{ return Helper.matchUndefShuffleVectorMask(*${root}); }]), (apply [{ Helper.replaceInstWithUndef(*${root}); }])>; -// Replace a G_SHUFFLE_VECTOR with a G_EXTRACT_VECTOR_ELT. -def shuffle_to_extract: GICombineRule< - (defs root:$root), - (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, - [{ return Helper.matchShuffleToExtract(*${root}); }]), - (apply [{ Helper.applyShuffleToExtract(*${root}); }])>; - // Replace an insert/extract element of an out of bounds index with undef. def insert_extract_vec_elt_out_of_bounds : GICombineRule< (defs root:$root), @@ -1674,8 +1667,7 @@ def combine_shuffle_concat : GICombineRule< // Combines shuffles of vector into build_vector def combine_shuffle_vector_to_build_vector : GICombineRule< (defs root:$root), - (match (G_SHUFFLE_VECTOR $dst, $src1, $src2, $mask):$root, - [{ return Helper.matchCombineShuffleToBuildVector(*${root}); }]), + (match (G_SHUFFLE_VECTOR $dst, $src1, $src2, $mask):$root), (apply [{ Helper.applyCombineShuffleToBuildVector(*${root}); }])>; def insert_vector_element_idx_undef : GICombineRule< diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index 2c2122a..bfd2817 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -118,9 +118,8 @@ enum CodeObjectVersionKind { class TargetOptions { public: TargetOptions() - : UnsafeFPMath(false), NoInfsFPMath(false), NoNaNsFPMath(false), - NoTrappingFPMath(true), NoSignedZerosFPMath(false), - EnableAIXExtendedAltivecABI(false), + : NoInfsFPMath(false), NoNaNsFPMath(false), NoTrappingFPMath(true), + NoSignedZerosFPMath(false), EnableAIXExtendedAltivecABI(false), HonorSignDependentRoundingFPMathOption(false), NoZerosInBSS(false), GuaranteedTailCallOpt(false), StackSymbolOrdering(true), EnableFastISel(false), EnableGlobalISel(false), UseInitArray(false), @@ -156,13 +155,6 @@ public: /// MCAsmInfo::BinutilsVersion. std::pair<int, int> BinutilsVersion{0, 0}; - /// UnsafeFPMath - This flag is enabled when the - /// -enable-unsafe-fp-math flag is specified on the command line. When - /// this flag is off (the default), the code generator is not allowed to - /// produce results that are "less precise" than IEEE allows. This includes - /// use of X86 instructions like FSIN and FCOS instead of libcalls. - unsigned UnsafeFPMath : 1; - /// NoInfsFPMath - This flag is enabled when the /// -enable-no-infs-fp-math flag is specified on the command line. When /// this flag is off (the default), the code generator is not allowed to diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h index 8e83b046..7e68ad2 100644 --- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h +++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h @@ -115,9 +115,9 @@ struct ArchInfo { // Defines the following partial order, indicating when an architecture is // a superset of another: // - // v9.6a > v9.5a > v9.4a > v9.3a > v9.2a > v9.1a > v9a; - // v v v v v - // v8.9a > v8.8a > v8.7a > v8.6a > v8.5a > v8.4a > ... > v8a; + // v9.7a > v9.6a > v9.5a > v9.4a > v9.3a > v9.2a > v9.1a > v9a; + // v v v v v + // v8.9a > v8.8a > v8.7a > v8.6a > v8.5a > ... > v8a; // // v8r has no relation to anything. This is used to determine which // features to enable for a given architecture. See diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.def b/llvm/include/llvm/TargetParser/ARMTargetParser.def index ff53aa1..0ada2e7 100644 --- a/llvm/include/llvm/TargetParser/ARMTargetParser.def +++ b/llvm/include/llvm/TargetParser/ARMTargetParser.def @@ -187,6 +187,11 @@ ARM_ARCH("armv9.6-a", ARMV9_6A, "9.6-A", "+v9.6a", ARMBuildAttrs::CPUArch::v9_A, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS | ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM)) +ARM_ARCH("armv9.7-a", ARMV9_7A, "9.7-A", "+v9.7a", ARMBuildAttrs::CPUArch::v9_A, + FK_NEON_FP_ARMV8, + (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | + ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS | + ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM)) ARM_ARCH("armv8-r", ARMV8R, "8-R", "+v8r", ARMBuildAttrs::CPUArch::v8_R, FK_FPV5_SP_D16, (ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h index dc8cd86d..0e82dd2 100644 --- a/llvm/include/llvm/TargetParser/Triple.h +++ b/llvm/include/llvm/TargetParser/Triple.h @@ -115,6 +115,7 @@ public: enum SubArchType { NoSubArch, + ARMSubArch_v9_7a, ARMSubArch_v9_6a, ARMSubArch_v9_5a, ARMSubArch_v9_4a, @@ -935,7 +936,8 @@ public: getEnvironment() == Triple::GNUEABIHF || getEnvironment() == Triple::GNUEABIHFT64 || getEnvironment() == Triple::OpenHOS || - getEnvironment() == Triple::MuslEABIHF || isAndroid()) && + getEnvironment() == Triple::MuslEABIHF || isOSFuchsia() || + isAndroid()) && isOSBinFormatELF() && !isOSNetBSD(); } diff --git a/llvm/include/llvm/Transforms/Instrumentation/AllocToken.h b/llvm/include/llvm/Transforms/Instrumentation/AllocToken.h index b1391cb0..077703c 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/AllocToken.h +++ b/llvm/include/llvm/Transforms/Instrumentation/AllocToken.h @@ -16,6 +16,7 @@ #include "llvm/IR/Analysis.h" #include "llvm/IR/PassManager.h" +#include "llvm/Support/AllocToken.h" #include <optional> namespace llvm { @@ -23,6 +24,7 @@ namespace llvm { class Module; struct AllocTokenOptions { + AllocTokenMode Mode = DefaultAllocTokenMode; std::optional<uint64_t> MaxTokens; bool FastABI = false; bool Extended = false; diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h index 979f3b3e..e677cbf 100644 --- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h +++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h @@ -21,6 +21,7 @@ #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Dominators.h" #include "llvm/Support/Compiler.h" +#include "llvm/Support/Printable.h" #include <cassert> namespace llvm { @@ -611,6 +612,10 @@ LLVM_ABI void InvertBranch(BranchInst *PBI, IRBuilderBase &Builder); // br/brcond/unreachable/ret LLVM_ABI bool hasOnlySimpleTerminator(const Function &F); +/// Print BasicBlock \p BB as an operand or print "<nullptr>" if \p BB is a +/// nullptr. +LLVM_ABI Printable printBasicBlock(const BasicBlock *BB); + } // end namespace llvm #endif // LLVM_TRANSFORMS_UTILS_BASICBLOCKUTILS_H |