diff options
Diffstat (limited to 'llvm/include')
27 files changed, 620 insertions, 172 deletions
diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h index 4a91b06..5b20d6bd 100644 --- a/llvm/include/llvm/ADT/STLExtras.h +++ b/llvm/include/llvm/ADT/STLExtras.h @@ -1692,6 +1692,28 @@ template <typename R, typename E> auto accumulate(R &&Range, E &&Init) { std::forward<E>(Init)); } +/// Wrapper for std::accumulate with a binary operator. +template <typename R, typename E, typename BinaryOp> +auto accumulate(R &&Range, E &&Init, BinaryOp &&Op) { + return std::accumulate(adl_begin(Range), adl_end(Range), + std::forward<E>(Init), std::forward<BinaryOp>(Op)); +} + +/// Returns the sum of all values in `Range` with `Init` initial value. +/// The default initial value is 0. +template <typename R, typename E = detail::ValueOfRange<R>> +auto sum_of(R &&Range, E Init = E{0}) { + return accumulate(std::forward<R>(Range), std::move(Init)); +} + +/// Returns the product of all values in `Range` with `Init` initial value. +/// The default initial value is 1. +template <typename R, typename E = detail::ValueOfRange<R>> +auto product_of(R &&Range, E Init = E{1}) { + return accumulate(std::forward<R>(Range), std::move(Init), + std::multiplies<>{}); +} + /// Provide wrappers to std::for_each which take ranges instead of having to /// pass begin/end explicitly. template <typename R, typename UnaryFunction> diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h index 8944e736..b9a08ce 100644 --- a/llvm/include/llvm/BinaryFormat/DXContainer.h +++ b/llvm/include/llvm/BinaryFormat/DXContainer.h @@ -201,19 +201,9 @@ enum class RootParameterType : uint32_t { LLVM_ABI ArrayRef<EnumEntry<RootParameterType>> getRootParameterTypes(); -#define ROOT_PARAMETER(Val, Enum) \ - case Val: \ - return true; -inline bool isValidParameterType(uint32_t V) { - switch (V) { -#include "DXContainerConstants.def" - } - return false; -} +bool isValidParameterType(uint32_t V); -inline bool isValidRangeType(uint32_t V) { - return V <= llvm::to_underlying(dxil::ResourceClass::LastEntry); -} +bool isValidRangeType(uint32_t V); #define SHADER_VISIBILITY(Val, Enum) Enum = Val, enum class ShaderVisibility : uint32_t { @@ -222,30 +212,14 @@ enum class ShaderVisibility : uint32_t { LLVM_ABI ArrayRef<EnumEntry<ShaderVisibility>> getShaderVisibility(); -#define SHADER_VISIBILITY(Val, Enum) \ - case Val: \ - return true; -inline bool isValidShaderVisibility(uint32_t V) { - switch (V) { -#include "DXContainerConstants.def" - } - return false; -} +bool isValidShaderVisibility(uint32_t V); #define FILTER(Val, Enum) Enum = Val, enum class SamplerFilter : uint32_t { #include "DXContainerConstants.def" }; -#define FILTER(Val, Enum) \ - case Val: \ - return true; -inline bool isValidSamplerFilter(uint32_t V) { - switch (V) { -#include "DXContainerConstants.def" - } - return false; -} +bool isValidSamplerFilter(uint32_t V); LLVM_ABI ArrayRef<EnumEntry<SamplerFilter>> getSamplerFilters(); @@ -256,15 +230,7 @@ enum class TextureAddressMode : uint32_t { LLVM_ABI ArrayRef<EnumEntry<TextureAddressMode>> getTextureAddressModes(); -#define TEXTURE_ADDRESS_MODE(Val, Enum) \ - case Val: \ - return true; -inline bool isValidAddress(uint32_t V) { - switch (V) { -#include "DXContainerConstants.def" - } - return false; -} +bool isValidAddress(uint32_t V); #define COMPARISON_FUNC(Val, Enum) Enum = Val, enum class ComparisonFunc : uint32_t { @@ -273,30 +239,20 @@ enum class ComparisonFunc : uint32_t { LLVM_ABI ArrayRef<EnumEntry<ComparisonFunc>> getComparisonFuncs(); -#define COMPARISON_FUNC(Val, Enum) \ - case Val: \ - return true; -inline bool isValidComparisonFunc(uint32_t V) { - switch (V) { -#include "DXContainerConstants.def" - } - return false; -} +bool isValidComparisonFunc(uint32_t V); #define STATIC_BORDER_COLOR(Val, Enum) Enum = Val, enum class StaticBorderColor : uint32_t { #include "DXContainerConstants.def" }; -#define STATIC_BORDER_COLOR(Val, Enum) \ - case Val: \ - return true; -inline bool isValidBorderColor(uint32_t V) { - switch (V) { -#include "DXContainerConstants.def" - } - return false; -} +bool isValidBorderColor(uint32_t V); + +bool isValidRootDesciptorFlags(uint32_t V); + +bool isValidDescriptorRangeFlags(uint32_t V); + +bool isValidStaticSamplerFlags(uint32_t V); LLVM_ABI ArrayRef<EnumEntry<StaticBorderColor>> getStaticBorderColors(); diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 1c7d346..464f475 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -800,6 +800,7 @@ enum AttributeKindCodes { ATTR_KIND_SANITIZE_TYPE = 101, ATTR_KIND_CAPTURES = 102, ATTR_KIND_DEAD_ON_RETURN = 103, + ATTR_KIND_SANITIZE_ALLOC_TOKEN = 104, }; enum ComdatSelectionKindCodes { diff --git a/llvm/include/llvm/CAS/OnDiskDataAllocator.h b/llvm/include/llvm/CAS/OnDiskDataAllocator.h new file mode 100644 index 0000000..2809df8 --- /dev/null +++ b/llvm/include/llvm/CAS/OnDiskDataAllocator.h @@ -0,0 +1,95 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// This file declares interface for OnDiskDataAllocator, a file backed data +/// pool can be used to allocate space to store data packed in a single file. It +/// is based on MappedFileRegionArena and includes a header in the beginning to +/// provide metadata. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CAS_ONDISKDATAALLOCATOR_H +#define LLVM_CAS_ONDISKDATAALLOCATOR_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/CAS/FileOffset.h" +#include "llvm/Support/Error.h" + +namespace llvm::cas { + +/// Sink for data. Stores variable length data with 8-byte alignment. Does not +/// track size of data, which is assumed to known from context, or embedded. +/// Uses 0-padding but does not guarantee 0-termination. +class OnDiskDataAllocator { +public: + using ValueProxy = MutableArrayRef<char>; + + /// A pointer to data stored on disk. + class OnDiskPtr { + public: + FileOffset getOffset() const { return Offset; } + explicit operator bool() const { return bool(getOffset()); } + const ValueProxy &operator*() const { + assert(Offset && "Null dereference"); + return Value; + } + const ValueProxy *operator->() const { + assert(Offset && "Null dereference"); + return &Value; + } + + OnDiskPtr() = default; + + private: + friend class OnDiskDataAllocator; + OnDiskPtr(FileOffset Offset, ValueProxy Value) + : Offset(Offset), Value(Value) {} + FileOffset Offset; + ValueProxy Value; + }; + + /// Get the data of \p Size stored at the given \p Offset. Note the allocator + /// doesn't keep track of the allocation size, thus \p Size doesn't need to + /// match the size of allocation but needs to be smaller. + Expected<ArrayRef<char>> get(FileOffset Offset, size_t Size) const; + + /// Allocate at least \p Size with 8-byte alignment. + Expected<OnDiskPtr> allocate(size_t Size); + + /// \returns the buffer that was allocated at \p create time, with size + /// \p UserHeaderSize. + MutableArrayRef<uint8_t> getUserHeader(); + + size_t size() const; + size_t capacity() const; + + static Expected<OnDiskDataAllocator> + create(const Twine &Path, const Twine &TableName, uint64_t MaxFileSize, + std::optional<uint64_t> NewFileInitialSize, + uint32_t UserHeaderSize = 0, + function_ref<void(void *)> UserHeaderInit = nullptr); + + OnDiskDataAllocator(OnDiskDataAllocator &&RHS); + OnDiskDataAllocator &operator=(OnDiskDataAllocator &&RHS); + + // No copy. Just call \a create() again. + OnDiskDataAllocator(const OnDiskDataAllocator &) = delete; + OnDiskDataAllocator &operator=(const OnDiskDataAllocator &) = delete; + + ~OnDiskDataAllocator(); + +private: + struct ImplType; + explicit OnDiskDataAllocator(std::unique_ptr<ImplType> Impl); + std::unique_ptr<ImplType> Impl; +}; + +} // namespace llvm::cas + +#endif // LLVM_CAS_ONDISKDATAALLOCATOR_H diff --git a/llvm/include/llvm/CAS/OnDiskTrieRawHashMap.h b/llvm/include/llvm/CAS/OnDiskTrieRawHashMap.h index 5e41bf6..fbd68d0 100644 --- a/llvm/include/llvm/CAS/OnDiskTrieRawHashMap.h +++ b/llvm/include/llvm/CAS/OnDiskTrieRawHashMap.h @@ -133,38 +133,38 @@ public: bool IsValue = false; }; - class pointer; - class const_pointer : public PointerImpl<ConstValueProxy> { + class OnDiskPtr; + class ConstOnDiskPtr : public PointerImpl<ConstValueProxy> { public: - const_pointer() = default; + ConstOnDiskPtr() = default; private: - friend class pointer; + friend class OnDiskPtr; friend class OnDiskTrieRawHashMap; - using const_pointer::PointerImpl::PointerImpl; + using ConstOnDiskPtr::PointerImpl::PointerImpl; }; - class pointer : public PointerImpl<ValueProxy> { + class OnDiskPtr : public PointerImpl<ValueProxy> { public: - operator const_pointer() const { - return const_pointer(Value, getOffset(), IsValue); + operator ConstOnDiskPtr() const { + return ConstOnDiskPtr(Value, getOffset(), IsValue); } - pointer() = default; + OnDiskPtr() = default; private: friend class OnDiskTrieRawHashMap; - using pointer::PointerImpl::PointerImpl; + using OnDiskPtr::PointerImpl::PointerImpl; }; /// Find the value from hash. /// /// \returns pointer to the value if exists, otherwise returns a non-value /// pointer that evaluates to `false` when convert to boolean. - const_pointer find(ArrayRef<uint8_t> Hash) const; + ConstOnDiskPtr find(ArrayRef<uint8_t> Hash) const; /// Helper function to recover a pointer into the trie from file offset. - Expected<const_pointer> recoverFromFileOffset(FileOffset Offset) const; + Expected<ConstOnDiskPtr> recoverFromFileOffset(FileOffset Offset) const; using LazyInsertOnConstructCB = function_ref<void(FileOffset TentativeOffset, ValueProxy TentativeValue)>; @@ -186,11 +186,11 @@ public: /// The in-memory \a TrieRawHashMap uses LazyAtomicPointer to synchronize /// simultaneous writes, but that seems dangerous to use in a memory-mapped /// file in case a process crashes in the busy state. - Expected<pointer> insertLazy(ArrayRef<uint8_t> Hash, - LazyInsertOnConstructCB OnConstruct = nullptr, - LazyInsertOnLeakCB OnLeak = nullptr); + Expected<OnDiskPtr> insertLazy(ArrayRef<uint8_t> Hash, + LazyInsertOnConstructCB OnConstruct = nullptr, + LazyInsertOnLeakCB OnLeak = nullptr); - Expected<pointer> insert(const ConstValueProxy &Value) { + Expected<OnDiskPtr> insert(const ConstValueProxy &Value) { return insertLazy(Value.Hash, [&](FileOffset, ValueProxy Allocated) { assert(Allocated.Hash == Value.Hash); assert(Allocated.Data.size() == Value.Data.size()); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h index 68bc54b..7c995a7 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h @@ -34,7 +34,7 @@ public: struct SymbolAddrs { ExecutorAddr Instance; ExecutorAddr Open; - ExecutorAddr Lookup; + ExecutorAddr Resolve; }; /// Create an EPCGenericMemoryAccess instance from a given set of @@ -51,25 +51,25 @@ public: LLVM_ABI Expected<tpctypes::DylibHandle> open(StringRef Path, uint64_t Mode); /// Looks up symbols within the given dylib. - Expected<std::vector<ExecutorSymbolDef>> - lookup(tpctypes::DylibHandle H, const SymbolLookupSet &Lookup) { - std::promise<MSVCPExpected<std::vector<ExecutorSymbolDef>>> RP; + Expected<tpctypes::LookupResult> lookup(tpctypes::DylibHandle H, + const SymbolLookupSet &Lookup) { + std::promise<MSVCPExpected<tpctypes::LookupResult>> RP; auto RF = RP.get_future(); lookupAsync(H, Lookup, [&RP](auto R) { RP.set_value(std::move(R)); }); return RF.get(); } /// Looks up symbols within the given dylib. - Expected<std::vector<ExecutorSymbolDef>> - lookup(tpctypes::DylibHandle H, const RemoteSymbolLookupSet &Lookup) { - std::promise<MSVCPExpected<std::vector<ExecutorSymbolDef>>> RP; + Expected<tpctypes::LookupResult> lookup(tpctypes::DylibHandle H, + const RemoteSymbolLookupSet &Lookup) { + std::promise<MSVCPExpected<tpctypes::LookupResult>> RP; auto RF = RP.get_future(); lookupAsync(H, Lookup, [&RP](auto R) { RP.set_value(std::move(R)); }); return RF.get(); } using SymbolLookupCompleteFn = - unique_function<void(Expected<std::vector<ExecutorSymbolDef>>)>; + unique_function<void(Expected<tpctypes::LookupResult>)>; /// Looks up symbols within the given dylib. LLVM_ABI void lookupAsync(tpctypes::DylibHandle H, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutorResolutionGenerator.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutorResolutionGenerator.h new file mode 100644 index 0000000..9b972ed --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutorResolutionGenerator.h @@ -0,0 +1,74 @@ +//===----- ExecutorResolver.h - Resolve symbols in executor -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Declares ExecutorResolutionGenerator for symbol resolution, +// dynamic library loading, and lookup in an executor process via +// ExecutorResolver. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_EXECUTORRESOLUTIONGENERATOR_H +#define LLVM_EXECUTIONENGINE_ORC_EXECUTORRESOLUTIONGENERATOR_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h" +#include "llvm/ExecutionEngine/Orc/Core.h" + +namespace llvm::orc { + +class ExecutorResolutionGenerator : public DefinitionGenerator { +public: + using SymbolPredicate = unique_function<bool(const SymbolStringPtr &)>; + using AbsoluteSymbolsFn = + unique_function<std::unique_ptr<MaterializationUnit>(SymbolMap)>; + + ExecutorResolutionGenerator( + ExecutionSession &ES, tpctypes::ResolverHandle H, + SymbolPredicate Allow = SymbolPredicate(), + AbsoluteSymbolsFn AbsoluteSymbols = absoluteSymbols) + : EPC(ES.getExecutorProcessControl()), H(H), Allow(std::move(Allow)), + AbsoluteSymbols(std::move(AbsoluteSymbols)) {} + + ExecutorResolutionGenerator( + ExecutionSession &ES, SymbolPredicate Allow = SymbolPredicate(), + AbsoluteSymbolsFn AbsoluteSymbols = absoluteSymbols) + : EPC(ES.getExecutorProcessControl()), Allow(std::move(Allow)), + AbsoluteSymbols(std::move(AbsoluteSymbols)) {} + + /// Permanently loads the library at the given path and, on success, returns + /// an ExecutorResolutionGenerator that will search it for symbol + /// definitions in the library. On failure returns the reason the library + /// failed to load. + static Expected<std::unique_ptr<ExecutorResolutionGenerator>> + Load(ExecutionSession &ES, const char *LibraryPath, + SymbolPredicate Allow = SymbolPredicate(), + AbsoluteSymbolsFn AbsoluteSymbols = absoluteSymbols); + + /// Creates a ExecutorResolutionGenerator that searches for symbols in + /// the target process. + static Expected<std::unique_ptr<ExecutorResolutionGenerator>> + GetForTargetProcess(ExecutionSession &ES, + SymbolPredicate Allow = SymbolPredicate(), + AbsoluteSymbolsFn AbsoluteSymbols = absoluteSymbols) { + return Load(ES, nullptr, std::move(Allow), std::move(AbsoluteSymbols)); + } + + Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &LookupSet) override; + +private: + ExecutorProcessControl &EPC; + tpctypes::ResolverHandle H; + SymbolPredicate Allow; + AbsoluteSymbolsFn AbsoluteSymbols; +}; + +} // namespace llvm::orc + +#endif // LLVM_EXECUTIONENGINE_ORC_EXECUTORRESOLUTIONGENERATOR_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h index 2bc6c12..99ba456 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h @@ -25,7 +25,7 @@ namespace rt { LLVM_ABI extern const char *SimpleExecutorDylibManagerInstanceName; LLVM_ABI extern const char *SimpleExecutorDylibManagerOpenWrapperName; -LLVM_ABI extern const char *SimpleExecutorDylibManagerLookupWrapperName; +LLVM_ABI extern const char *SimpleExecutorDylibManagerResolveWrapperName; LLVM_ABI extern const char *SimpleExecutorMemoryManagerInstanceName; LLVM_ABI extern const char *SimpleExecutorMemoryManagerReserveWrapperName; @@ -66,10 +66,9 @@ using SPSSimpleExecutorDylibManagerOpenSignature = shared::SPSExpected<shared::SPSExecutorAddr>(shared::SPSExecutorAddr, shared::SPSString, uint64_t); -using SPSSimpleExecutorDylibManagerLookupSignature = - shared::SPSExpected<shared::SPSSequence<shared::SPSExecutorSymbolDef>>( - shared::SPSExecutorAddr, shared::SPSExecutorAddr, - shared::SPSRemoteSymbolLookupSet); +using SPSSimpleExecutorDylibManagerResolveSignature = shared::SPSExpected< + shared::SPSSequence<shared::SPSOptional<shared::SPSExecutorSymbolDef>>>( + shared::SPSExecutorAddr, shared::SPSRemoteSymbolLookupSet); using SPSSimpleExecutorMemoryManagerReserveSignature = shared::SPSExpected<shared::SPSExecutorAddr>(shared::SPSExecutorAddr, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h index adb07ba..28ff322 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h @@ -114,7 +114,11 @@ struct PointerWrite { /// A handle used to represent a loaded dylib in the target process. using DylibHandle = ExecutorAddr; -using LookupResult = std::vector<ExecutorSymbolDef>; +/// A handle used to reference the resolver associated with a loaded +/// dylib in the target process. +using ResolverHandle = ExecutorAddr; + +using LookupResult = std::vector<std::optional<ExecutorSymbolDef>>; } // end namespace tpctypes diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorResolver.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorResolver.h new file mode 100644 index 0000000..2c5e98c --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorResolver.h @@ -0,0 +1,48 @@ +//===----- ExecutorResolver.h - Symbol resolver -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Executor Symbol resolver. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORRESOLVER_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORRESOLVER_H + +#include "llvm/ADT/FunctionExtras.h" + +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h" +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" + +namespace llvm::orc { + +class ExecutorResolver { +public: + using ResolveResult = Expected<std::vector<std::optional<ExecutorSymbolDef>>>; + using YieldResolveResultFn = unique_function<void(ResolveResult)>; + + virtual ~ExecutorResolver() = default; + + virtual void resolveAsync(const RemoteSymbolLookupSet &L, + YieldResolveResultFn &&OnResolve) = 0; +}; + +class DylibSymbolResolver : public ExecutorResolver { +public: + DylibSymbolResolver(tpctypes::DylibHandle H) : Handle(H) {} + + void + resolveAsync(const RemoteSymbolLookupSet &L, + ExecutorResolver::YieldResolveResultFn &&OnResolve) override; + +private: + tpctypes::DylibHandle Handle; +}; + +} // end namespace llvm::orc +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORRESOLVER_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h index 36a6f4b..7526a29d 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h @@ -23,6 +23,7 @@ #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorResolver.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/Error.h" @@ -39,8 +40,6 @@ public: virtual ~SimpleExecutorDylibManager(); Expected<tpctypes::DylibHandle> open(const std::string &Path, uint64_t Mode); - Expected<std::vector<ExecutorSymbolDef>> - lookup(tpctypes::DylibHandle H, const RemoteSymbolLookupSet &L); Error shutdown() override; void addBootstrapSymbols(StringMap<ExecutorAddr> &M) override; @@ -52,10 +51,11 @@ private: openWrapper(const char *ArgData, size_t ArgSize); static llvm::orc::shared::CWrapperFunctionResult - lookupWrapper(const char *ArgData, size_t ArgSize); + resolveWrapper(const char *ArgData, size_t ArgSize); std::mutex M; DylibSet Dylibs; + std::vector<std::unique_ptr<ExecutorResolver>> Resolvers; }; } // end namespace rt_bootstrap diff --git a/llvm/include/llvm/Frontend/HLSL/RootSignatureValidations.h b/llvm/include/llvm/Frontend/HLSL/RootSignatureValidations.h index 4dd1811..7131980 100644 --- a/llvm/include/llvm/Frontend/HLSL/RootSignatureValidations.h +++ b/llvm/include/llvm/Frontend/HLSL/RootSignatureValidations.h @@ -28,12 +28,14 @@ LLVM_ABI bool verifyRootFlag(uint32_t Flags); LLVM_ABI bool verifyVersion(uint32_t Version); LLVM_ABI bool verifyRegisterValue(uint32_t RegisterValue); LLVM_ABI bool verifyRegisterSpace(uint32_t RegisterSpace); -LLVM_ABI bool verifyRootDescriptorFlag(uint32_t Version, uint32_t FlagsVal); +LLVM_ABI bool verifyRootDescriptorFlag(uint32_t Version, + dxbc::RootDescriptorFlags Flags); LLVM_ABI bool verifyRangeType(uint32_t Type); LLVM_ABI bool verifyDescriptorRangeFlag(uint32_t Version, dxil::ResourceClass Type, - dxbc::DescriptorRangeFlags FlagsVal); -LLVM_ABI bool verifyStaticSamplerFlags(uint32_t Version, uint32_t FlagsNumber); + dxbc::DescriptorRangeFlags Flags); +LLVM_ABI bool verifyStaticSamplerFlags(uint32_t Version, + dxbc::StaticSamplerFlags Flags); LLVM_ABI bool verifyNumDescriptors(uint32_t NumDescriptors); LLVM_ABI bool verifyMipLODBias(float MipLODBias); LLVM_ABI bool verifyMaxAnisotropy(uint32_t MaxAnisotropy); diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h index 6e1bce1..7bec7e0 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h @@ -239,6 +239,9 @@ enum class OpenMPOffloadMappingFlags : uint64_t { // dynamic. // This is an OpenMP extension for the sake of OpenACC support. OMP_MAP_OMPX_HOLD = 0x2000, + // Attach pointer and pointee, after processing all other maps. + // Applicable to map-entering directives. Does not change ref-count. + OMP_MAP_ATTACH = 0x4000, /// Signal that the runtime library should use args as an array of /// descriptor_dim pointers and use args_size as dims. Used when we have /// non-contiguous list items in target update directive diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td index ef816fb..8e7d9dc 100644 --- a/llvm/include/llvm/IR/Attributes.td +++ b/llvm/include/llvm/IR/Attributes.td @@ -342,6 +342,9 @@ def SanitizeRealtime : EnumAttr<"sanitize_realtime", IntersectPreserve, [FnAttr] /// during a real-time sanitized function (see `sanitize_realtime`). def SanitizeRealtimeBlocking : EnumAttr<"sanitize_realtime_blocking", IntersectPreserve, [FnAttr]>; +/// Allocation token instrumentation is on. +def SanitizeAllocToken : EnumAttr<"sanitize_alloc_token", IntersectPreserve, [FnAttr]>; + /// Speculative Load Hardening is enabled. /// /// Note that this uses the default compatibility (always compatible during diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h index 6652e30..7c6e709 100644 --- a/llvm/include/llvm/IR/DebugInfoMetadata.h +++ b/llvm/include/llvm/IR/DebugInfoMetadata.h @@ -2600,14 +2600,19 @@ public: StringRef getDirectory() const { return getScope()->getDirectory(); } std::optional<StringRef> getSource() const { return getScope()->getSource(); } - /// Get the scope where this is inlined. - /// - /// Walk through \a getInlinedAt() and return \a getScope() from the deepest - /// location. + /// Walk through \a getInlinedAt() and return the \a DILocation of the + /// outermost call site in the inlining chain. + const DILocation *getInlinedAtLocation() const { + const DILocation *Current = this; + while (const DILocation *Next = Current->getInlinedAt()) + Current = Next; + return Current; + } + + // Return the \a DILocalScope of the outermost call site in the inlining + // chain. DILocalScope *getInlinedAtScope() const { - if (auto *IA = getInlinedAt()) - return IA->getInlinedAtScope(); - return getScope(); + return getInlinedAtLocation()->getScope(); } /// Get the DWARF discriminator. diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def index 0603abc..74746cc 100644 --- a/llvm/include/llvm/IR/FixedMetadataKinds.def +++ b/llvm/include/llvm/IR/FixedMetadataKinds.def @@ -56,3 +56,4 @@ LLVM_FIXED_MD_KIND(MD_noalias_addrspace, "noalias.addrspace", 41) LLVM_FIXED_MD_KIND(MD_callee_type, "callee_type", 42) LLVM_FIXED_MD_KIND(MD_nofree, "nofree", 43) LLVM_FIXED_MD_KIND(MD_captures, "captures", 44) +LLVM_FIXED_MD_KIND(MD_alloc_token, "alloc_token", 45) diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h index 783f8f6..041a4ce 100644 --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -1722,16 +1722,19 @@ public: return Insert(BinOp, Name); } - Value *CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name = "") { + Value *CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name = "", + Instruction *MDFrom = nullptr) { assert(Cond2->getType()->isIntOrIntVectorTy(1)); return CreateSelect(Cond1, Cond2, - ConstantInt::getNullValue(Cond2->getType()), Name); + ConstantInt::getNullValue(Cond2->getType()), Name, + MDFrom); } - Value *CreateLogicalOr(Value *Cond1, Value *Cond2, const Twine &Name = "") { + Value *CreateLogicalOr(Value *Cond1, Value *Cond2, const Twine &Name = "", + Instruction *MDFrom = nullptr) { assert(Cond2->getType()->isIntOrIntVectorTy(1)); return CreateSelect(Cond1, ConstantInt::getAllOnesValue(Cond2->getType()), - Cond2, Name); + Cond2, Name, MDFrom); } Value *CreateLogicalOp(Instruction::BinaryOps Opc, Value *Cond1, Value *Cond2, diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td index 23d878f..3af1750 100644 --- a/llvm/include/llvm/IR/IntrinsicsNVVM.td +++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td @@ -272,6 +272,10 @@ class WMMA_REGS<string Geom, string Frag, string PtxEltType, bit IsSparse = fals !eq(gft,"m16n8k16:d:f32") : !listsplat(llvm_float_ty, 4), !eq(gft,"m16n8k4:c:f32") : !listsplat(llvm_float_ty, 4), !eq(gft,"m16n8k4:d:f32") : !listsplat(llvm_float_ty, 4), + !eq(gft,"m16n8k32:c:f16") : !listsplat(llvm_v2f16_ty, 2), + !eq(gft,"m16n8k32:c:f32") : !listsplat(llvm_float_ty, 4), + !eq(gft,"m16n8k32:d:f16") : !listsplat(llvm_v2f16_ty, 2), + !eq(gft,"m16n8k32:d:f32") : !listsplat(llvm_float_ty, 4), // wmma fp16 -> fp16/fp32 @ m16n16k16/m8n32k16/m32n8k16 // All other supported geometries use the same fragment format for f32 and @@ -298,6 +302,21 @@ class WMMA_REGS<string Geom, string Frag, string PtxEltType, bit IsSparse = fals !eq(gft,"m8n8k4:c:f64") : !listsplat(llvm_double_ty, 2), !eq(gft,"m8n8k4:d:f64") : !listsplat(llvm_double_ty, 2), + !eq(gft,"m16n8k4:a:f64") : !listsplat(llvm_double_ty, 2), + !eq(gft,"m16n8k4:b:f64") : [llvm_double_ty], + !eq(gft,"m16n8k4:c:f64") : !listsplat(llvm_double_ty, 4), + !eq(gft,"m16n8k4:d:f64") : !listsplat(llvm_double_ty, 4), + + !eq(gft,"m16n8k8:a:f64") : !listsplat(llvm_double_ty, 4), + !eq(gft,"m16n8k8:b:f64") : !listsplat(llvm_double_ty, 2), + !eq(gft,"m16n8k8:c:f64") : !listsplat(llvm_double_ty, 4), + !eq(gft,"m16n8k8:d:f64") : !listsplat(llvm_double_ty, 4), + + !eq(gft,"m16n8k16:a:f64") : !listsplat(llvm_double_ty, 8), + !eq(gft,"m16n8k16:b:f64") : !listsplat(llvm_double_ty, 4), + !eq(gft,"m16n8k16:c:f64") : !listsplat(llvm_double_ty, 4), + !eq(gft,"m16n8k16:d:f64") : !listsplat(llvm_double_ty, 4), + // wmma bf16 -> s32 @ m16n16k16/m8n32k16/m32n8k16 !eq(gft,"m16n16k16:a:bf16") : !listsplat(llvm_i32_ty, 4), !eq(gft,"m16n16k16:b:bf16") : !listsplat(llvm_i32_ty, 4), @@ -378,6 +397,26 @@ class WMMA_REGS<string Geom, string Frag, string PtxEltType, bit IsSparse = fals !eq(gft,"m16n8k64:c:s32") : !listsplat(llvm_i32_ty, 4), !eq(gft,"m16n8k64:d:s32") : !listsplat(llvm_i32_ty, 4), + // mma e4m3/e5m2 -> f16/f32 @ m16n8k16 + !eq(gft,"m16n8k16:a:e4m3") : !listsplat(llvm_i32_ty, 2), + !eq(gft,"m16n8k16:a:e5m2") : !listsplat(llvm_i32_ty, 2), + !eq(gft,"m16n8k16:b:e4m3") : [llvm_i32_ty], + !eq(gft,"m16n8k16:b:e5m2") : [llvm_i32_ty], + // mma e4m3/e5m2/e3m2/e2m3/e2m1 -> f32 @ m16n8k32 + !eq(gft,"m16n8k32:a:e4m3") : !listsplat(llvm_i32_ty, 4), + !eq(gft,"m16n8k32:a:e5m2") : !listsplat(llvm_i32_ty, 4), + !eq(gft,"m16n8k32:a:e3m2") : !listsplat(llvm_i32_ty, 4), + !eq(gft,"m16n8k32:a:e2m3") : !listsplat(llvm_i32_ty, 4), + !eq(gft,"m16n8k32:a:e2m1") : !listsplat(llvm_i32_ty, 4), + !eq(gft,"m16n8k32:b:e4m3") : !listsplat(llvm_i32_ty, 2), + !eq(gft,"m16n8k32:b:e5m2") : !listsplat(llvm_i32_ty, 2), + !eq(gft,"m16n8k32:b:e3m2") : !listsplat(llvm_i32_ty, 2), + !eq(gft,"m16n8k32:b:e2m3") : !listsplat(llvm_i32_ty, 2), + !eq(gft,"m16n8k32:b:e2m1") : !listsplat(llvm_i32_ty, 2), + // mma e2m1 -> f32 @m16n8k64 + !eq(gft,"m16n8k64:a:e2m1") : !listsplat(llvm_i32_ty, 4), + !eq(gft,"m16n8k64:b:e2m1") : !listsplat(llvm_i32_ty, 2), + // wmma/mma b1 -> s32 @ m8n8k128(b1) !eq(gft,"m8n8k128:a:b1") : [llvm_i32_ty], !eq(gft,"m8n8k128:b:b1") : [llvm_i32_ty], @@ -468,7 +507,7 @@ class WMMA_NAME<string ALayout, string BLayout, int Satfinite, string Rnd, strin # !if(Satfinite, "_satfinite", ""); } -class MMA_NAME<string ALayout, string BLayout, int Satfinite, string b1op, +class MMA_NAME<string ALayout, string BLayout, int Satfinite, string b1op, string Kind, WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> { string signature = MMA_SIGNATURE<A, B, C, D>.ret; string record = "int_nvvm_mma" @@ -476,6 +515,7 @@ class MMA_NAME<string ALayout, string BLayout, int Satfinite, string b1op, # "_" # A.geom # "_" # ALayout # "_" # BLayout + # !if(!ne(Kind, ""), !strconcat("_", !subst("::", "_", Kind)), "") # !if(Satfinite, "_satfinite", "") # signature; } @@ -601,7 +641,7 @@ class NVVM_MMA_OPS { ["m16n8k16", "m16n8k8"], ["bf16"], [], ["f32"], []>.ret; list<list<WMMA_REGS>> f64_mma_ops = MMA_OPS< - ["m8n8k4"], + ["m8n8k4", "m16n8k4", "m16n8k8", "m16n8k16"], ["f64"], [], ["f64"], []>.ret; list<list<WMMA_REGS>> fp_mma_ops = MMA_OPS< ["m8n8k4", "m16n8k8", "m16n8k16"], @@ -609,6 +649,18 @@ class NVVM_MMA_OPS { list<list<WMMA_REGS>> int_mma_ops = MMA_OPS< ["m8n8k16", "m16n8k16", "m16n8k32"], ["s8", "u8"], ["s8", "u8"], ["s32"], []>.ret; + // m16n8k32 fp8 variants are intersected with f8f6f4 variants + // and processed there + list<list<WMMA_REGS>> fp8_mma_ops = MMA_OPS< + ["m16n8k16"], + ["e4m3", "e5m2"], ["e4m3", "e5m2"], + ["f16", "f32"], ["f16", "f32"]>.ret; + // it also contains e4m3/e5m2 from fp8 variants + list<list<WMMA_REGS>> f8f6f4_mma_ops = MMA_OPS< + ["m16n8k32"], + ["e4m3", "e5m2", "e3m2", "e2m3", "e2m1"], + ["e4m3", "e5m2", "e3m2", "e2m3", "e2m1"], + ["f16", "f32"], ["f16", "f32"]>.ret; list<list<WMMA_REGS>> subint_mma_ops = MMA_OPS< ["m8n8k32", "m16n8k32", "m16n8k64"], ["s4", "u4"], ["s4", "u4"], ["s32"], []>.ret; @@ -617,7 +669,8 @@ class NVVM_MMA_OPS { ["b1"], [], ["s32"], []>.ret; list<list<WMMA_REGS>> all_mma_ops = !listconcat( tf32_mma_ops, bf16_mma_ops, f64_mma_ops, - fp_mma_ops, int_mma_ops, subint_mma_ops, bit_mma_ops); + fp_mma_ops, fp8_mma_ops, f8f6f4_mma_ops, + int_mma_ops, subint_mma_ops, bit_mma_ops); list<list<WMMA_REGS>> bf16_mma_sp_ops = MMA_OPS< ["m16n8k16", "m16n8k32"], @@ -770,7 +823,8 @@ class NVVM_MMA_B1OPS<list<WMMA_REGS> frags> { // if NVVM_MMA_SUPPORTED<...>.ret then // def : FOO<>; // The record will only be defined for supported ops. // -class NVVM_MMA_SUPPORTED<list<WMMA_REGS> frags, string layout_a, string layout_b, int satf> { +class NVVM_MMA_SUPPORTED<list<WMMA_REGS> frags, string layout_a, string layout_b, + string kind, int satf> { // MMA ops check both layouts. string layout = layout_a # ":" # layout_b; string a_type = frags[0].ptx_elt_type; @@ -805,10 +859,31 @@ class NVVM_MMA_SUPPORTED<list<WMMA_REGS> frags, string layout_a, string layout_b !or(!ne(a_type, b_type), !ne(c_type, d_type))): false, - // m16n8k8 requires C and D to be the same type. - !and(!eq(geom, "m16n8k8"), + // m16n8k16/m16n8k32 requires C and D to be the same type + !and(!or(!eq(geom, "m16n8k16"), + !eq(geom, "m16n8k32")), !ne(c_type, d_type)): false, + // Limit kind to valid types and geometries + !and(!ne(kind, ""), + !or(!ne(geom, "m16n8k32"), + !and(!ne(a_type, "e4m3"), + !ne(a_type, "e5m2"), + !ne(a_type, "e3m2"), + !ne(a_type, "e2m3"), + !ne(a_type, "e2m1")))): false, + + // Limit m16n8k16/m16n8k32 with no kind to valid types + !and(!eq(kind, ""), + !or(!eq(geom, "m16n8k16"), + !eq(geom, "m16n8k32")), + !or(!eq(a_type, "e3m2"), + !eq(a_type, "e2m3"), + !eq(a_type, "e2m1"), + !eq(b_type, "e3m2"), + !eq(b_type, "e2m3"), + !eq(b_type, "e2m1"))): false, + // All other are OK. true: true ); @@ -882,9 +957,10 @@ class NVVM_MMA_SP_SUPPORTED<list<WMMA_REGS> frags, string metadata, !eq(a_type, "tf32")), !ne(a_type, b_type)): false, - // m16n8k16 and m16n8k32 requires C and D to be the same type. + // m16n8k16, m16n8k32 and m16n8k64 requires C and D to be the same type. !and(!or(!eq(geom, "m16n8k16"), - !eq(geom, "m16n8k32")), + !eq(geom, "m16n8k32"), + !eq(geom, "m16n8k64")), !ne(c_type, d_type)): false, !and(!eq(kind, ""), @@ -2252,10 +2328,12 @@ foreach layout_a = ["row", "col"] in { foreach satf = [0, 1] in { foreach op = NVVM_MMA_OPS.all_mma_ops in { foreach b1op = NVVM_MMA_B1OPS<op>.ret in { - if NVVM_MMA_SUPPORTED<op, layout_a, layout_b, satf>.ret then { - def MMA_NAME<layout_a, layout_b, satf, b1op, op[0], op[1], op[2], op[3]>.record - : NVVM_MMA<op[0], op[1], op[2], op[3]>; - } + foreach kind = ["", "kind::f8f6f4"] in { + if NVVM_MMA_SUPPORTED<op, layout_a, layout_b, kind, satf>.ret then { + def MMA_NAME<layout_a, layout_b, satf, b1op, kind, op[0], op[1], op[2], op[3]>.record + : NVVM_MMA<op[0], op[1], op[2], op[3]>; + } + } // kind } // b1op } // op } // satf diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td index 823c491..66e24fa 100644 --- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td +++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td @@ -150,6 +150,14 @@ def int_spv_rsqrt : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty] [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [IntrNoMem]>; + def int_spv_resource_counterhandlefromimplicitbinding + : DefaultAttrsIntrinsic<[llvm_any_ty], + [llvm_any_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_spv_resource_counterhandlefrombinding + : DefaultAttrsIntrinsic<[llvm_any_ty], + [llvm_any_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem]>; def int_spv_firstbituhigh : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_anyint_ty], [IntrNoMem]>; def int_spv_firstbitshigh : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_anyint_ty], [IntrNoMem]>; diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h index 570d4c0..234c587 100644 --- a/llvm/include/llvm/MC/TargetRegistry.h +++ b/llvm/include/llvm/MC/TargetRegistry.h @@ -737,7 +737,8 @@ struct TargetRegistry { /// \param TripleStr - The triple to use for finding a target. /// \param Error - On failure, an error string describing why no target was /// found. - // TODO: Drop this in favor of the method accepting Triple. + // TODO(boomanaiden154): Remove this function after LLVM 22 branches. + [[deprecated("Use overload accepting Triple instead")]] static const Target *lookupTarget(StringRef TripleStr, std::string &Error) { return lookupTarget(Triple(TripleStr), Error); } diff --git a/llvm/include/llvm/Object/OffloadBundle.h b/llvm/include/llvm/Object/OffloadBundle.h index 18be62b..bbb313c0 100644 --- a/llvm/include/llvm/Object/OffloadBundle.h +++ b/llvm/include/llvm/Object/OffloadBundle.h @@ -32,29 +32,41 @@ namespace llvm { namespace object { +// CompressedOffloadBundle represents the format for the compressed offload +// bundles. +// +// The format is as follows: +// - Magic Number (4 bytes) - A constant "CCOB". +// - Version (2 bytes) +// - Compression Method (2 bytes) - Uses the values from +// llvm::compression::Format. +// - Total file size (4 bytes in V2, 8 bytes in V3). +// - Uncompressed Size (4 bytes in V1/V2, 8 bytes in V3). +// - Truncated MD5 Hash (8 bytes). +// - Compressed Data (variable length). class CompressedOffloadBundle { private: - static inline const size_t MagicSize = 4; - static inline const size_t VersionFieldSize = sizeof(uint16_t); - static inline const size_t MethodFieldSize = sizeof(uint16_t); - static inline const size_t FileSizeFieldSize = sizeof(uint32_t); - static inline const size_t UncompressedSizeFieldSize = sizeof(uint32_t); - static inline const size_t HashFieldSize = sizeof(uint64_t); - static inline const size_t V1HeaderSize = - MagicSize + VersionFieldSize + MethodFieldSize + - UncompressedSizeFieldSize + HashFieldSize; - static inline const size_t V2HeaderSize = - MagicSize + VersionFieldSize + FileSizeFieldSize + MethodFieldSize + - UncompressedSizeFieldSize + HashFieldSize; static inline const llvm::StringRef MagicNumber = "CCOB"; - static inline const uint16_t Version = 2; public: - LLVM_ABI static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>> + struct CompressedBundleHeader { + unsigned Version; + llvm::compression::Format CompressionFormat; + std::optional<size_t> FileSize; + size_t UncompressedFileSize; + uint64_t Hash; + + static llvm::Expected<CompressedBundleHeader> tryParse(llvm::StringRef); + }; + + static inline const uint16_t DefaultVersion = 3; + + static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>> compress(llvm::compression::Params P, const llvm::MemoryBuffer &Input, - bool Verbose = false); - LLVM_ABI static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>> - decompress(llvm::MemoryBufferRef &Input, bool Verbose = false); + uint16_t Version, raw_ostream *VerboseStream = nullptr); + static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>> + decompress(const llvm::MemoryBuffer &Input, + raw_ostream *VerboseStream = nullptr); }; /// Bundle entry in binary clang-offload-bundler format. @@ -62,12 +74,12 @@ struct OffloadBundleEntry { uint64_t Offset = 0u; uint64_t Size = 0u; uint64_t IDLength = 0u; - StringRef ID; + std::string ID; OffloadBundleEntry(uint64_t O, uint64_t S, uint64_t I, StringRef T) - : Offset(O), Size(S), IDLength(I), ID(T) {} + : Offset(O), Size(S), IDLength(I), ID(T.str()) {} void dumpInfo(raw_ostream &OS) { OS << "Offset = " << Offset << ", Size = " << Size - << ", ID Length = " << IDLength << ", ID = " << ID; + << ", ID Length = " << IDLength << ", ID = " << ID << "\n"; } void dumpURI(raw_ostream &OS, StringRef FilePath) { OS << ID.data() << "\tfile://" << FilePath << "#offset=" << Offset @@ -81,16 +93,21 @@ class OffloadBundleFatBin { uint64_t Size = 0u; StringRef FileName; uint64_t NumberOfEntries; + bool Decompressed; SmallVector<OffloadBundleEntry> Entries; public: + std::unique_ptr<MemoryBuffer> DecompressedBuffer; + SmallVector<OffloadBundleEntry> getEntries() { return Entries; } uint64_t getSize() const { return Size; } StringRef getFileName() const { return FileName; } uint64_t getNumEntries() const { return NumberOfEntries; } + bool isDecompressed() const { return Decompressed; } LLVM_ABI static Expected<std::unique_ptr<OffloadBundleFatBin>> - create(MemoryBufferRef, uint64_t SectionOffset, StringRef FileName); + create(MemoryBufferRef, uint64_t SectionOffset, StringRef FileName, + bool Decompress = false); LLVM_ABI Error extractBundle(const ObjectFile &Source); LLVM_ABI Error dumpEntryToCodeObject(); @@ -106,9 +123,14 @@ public: Entry.dumpURI(outs(), FileName); } - OffloadBundleFatBin(MemoryBufferRef Source, StringRef File) - : FileName(File), NumberOfEntries(0), - Entries(SmallVector<OffloadBundleEntry>()) {} + OffloadBundleFatBin(MemoryBufferRef Source, StringRef File, + bool Decompress = false) + : FileName(File), NumberOfEntries(0), Decompressed(Decompress), + Entries(SmallVector<OffloadBundleEntry>()) { + if (Decompress) + DecompressedBuffer = + MemoryBuffer::getMemBufferCopy(Source.getBuffer(), File); + } }; enum UriTypeT { FILE_URI, MEMORY_URI }; @@ -191,6 +213,10 @@ LLVM_ABI Error extractOffloadBundleFatBinary( LLVM_ABI Error extractCodeObject(const ObjectFile &Source, int64_t Offset, int64_t Size, StringRef OutputFileName); +/// Extract code object memory from the given \p Source object file at \p Offset +/// and of \p Size, and copy into \p OutputFileName. +LLVM_ABI Error extractCodeObject(MemoryBufferRef Buffer, int64_t Offset, + int64_t Size, StringRef OutputFileName); /// Extracts an Offload Bundle Entry given by URI LLVM_ABI Error extractOffloadBundleByURI(StringRef URIstr); diff --git a/llvm/include/llvm/Option/ArgList.h b/llvm/include/llvm/Option/ArgList.h index 3e80574..db36509 100644 --- a/llvm/include/llvm/Option/ArgList.h +++ b/llvm/include/llvm/Option/ArgList.h @@ -20,6 +20,7 @@ #include "llvm/Option/OptSpecifier.h" #include "llvm/Option/Option.h" #include "llvm/Support/Compiler.h" +#include "llvm/Support/Error.h" #include <algorithm> #include <cstddef> #include <initializer_list> @@ -280,6 +281,22 @@ public: /// list. virtual unsigned getNumInputArgStrings() const = 0; + /// getSubCommand - Find subcommand from the arguments if the usage is valid. + /// + /// \param AllSubCommands - A list of all valid subcommands. + /// \param HandleMultipleSubcommands - A callback for the case where multiple + /// subcommands are present in the arguments. It gets a list of all found + /// subcommands. + /// \param HandleOtherPositionals - A callback for the case where positional + /// arguments that are not subcommands are present. + /// \return The name of the subcommand found. If no subcommand is found, + /// this returns an empty StringRef. If multiple subcommands are found, the + /// first one is returned. + StringRef getSubCommand( + ArrayRef<OptTable::SubCommand> AllSubCommands, + std::function<void(ArrayRef<StringRef>)> HandleMultipleSubcommands, + std::function<void(ArrayRef<StringRef>)> HandleOtherPositionals) const; + /// @} /// @name Argument Lookup Utilities /// @{ diff --git a/llvm/include/llvm/Option/OptParser.td b/llvm/include/llvm/Option/OptParser.td index 9fd606b..8f32fb4 100644 --- a/llvm/include/llvm/Option/OptParser.td +++ b/llvm/include/llvm/Option/OptParser.td @@ -98,7 +98,15 @@ class HelpTextVariant<list<OptionVisibility> visibilities, string text> { string Text = text; } -class Option<list<string> prefixes, string name, OptionKind kind> { +// Class definition for positional subcommands. +class SubCommand<string name, string helpText, string usage = ""> { + string Name = name; + string HelpText = helpText; + string Usage = usage; +} + +class Option<list<string> prefixes, string name, OptionKind kind, + list<SubCommand> subcommands = []> { string EnumName = ?; // Uses the def name if undefined. list<string> Prefixes = prefixes; string Name = name; @@ -129,26 +137,34 @@ class Option<list<string> prefixes, string name, OptionKind kind> { code ValueMerger = "mergeForwardValue"; code ValueExtractor = "extractForwardValue"; list<code> NormalizedValues = ?; + list<SubCommand> SubCommands = subcommands; } // Helpers for defining options. -class Flag<list<string> prefixes, string name> - : Option<prefixes, name, KIND_FLAG>; -class Joined<list<string> prefixes, string name> - : Option<prefixes, name, KIND_JOINED>; -class Separate<list<string> prefixes, string name> - : Option<prefixes, name, KIND_SEPARATE>; -class CommaJoined<list<string> prefixes, string name> - : Option<prefixes, name, KIND_COMMAJOINED>; -class MultiArg<list<string> prefixes, string name, int numargs> - : Option<prefixes, name, KIND_MULTIARG> { +class Flag<list<string> prefixes, string name, + list<SubCommand> subcommands = []> + : Option<prefixes, name, KIND_FLAG, subcommands>; +class Joined<list<string> prefixes, string name, + list<SubCommand> subcommands = []> + : Option<prefixes, name, KIND_JOINED, subcommands>; +class Separate<list<string> prefixes, string name, + list<SubCommand> subcommands = []> + : Option<prefixes, name, KIND_SEPARATE, subcommands>; +class CommaJoined<list<string> prefixes, string name, + list<SubCommand> subcommands = []> + : Option<prefixes, name, KIND_COMMAJOINED, subcommands>; +class MultiArg<list<string> prefixes, string name, int numargs, + list<SubCommand> subcommands = []> + : Option<prefixes, name, KIND_MULTIARG, subcommands> { int NumArgs = numargs; } -class JoinedOrSeparate<list<string> prefixes, string name> - : Option<prefixes, name, KIND_JOINED_OR_SEPARATE>; -class JoinedAndSeparate<list<string> prefixes, string name> - : Option<prefixes, name, KIND_JOINED_AND_SEPARATE>; +class JoinedOrSeparate<list<string> prefixes, string name, + list<SubCommand> subcommands = []> + : Option<prefixes, name, KIND_JOINED_OR_SEPARATE, subcommands>; +class JoinedAndSeparate<list<string> prefixes, string name, + list<SubCommand> subcommands = []> + : Option<prefixes, name, KIND_JOINED_AND_SEPARATE, subcommands>; // Mix-ins for adding optional attributes. diff --git a/llvm/include/llvm/Option/OptTable.h b/llvm/include/llvm/Option/OptTable.h index df42ee3..f641ca4 100644 --- a/llvm/include/llvm/Option/OptTable.h +++ b/llvm/include/llvm/Option/OptTable.h @@ -53,6 +53,13 @@ public: /// parts of the driver still use Option instances where convenient. class LLVM_ABI OptTable { public: + /// Represents a subcommand and its options in the option table. + struct SubCommand { + const char *Name; + const char *HelpText; + const char *Usage; + }; + /// Entry for a single option instance in the option data table. struct Info { unsigned PrefixesOffset; @@ -79,6 +86,8 @@ public: unsigned short AliasID; const char *AliasArgs; const char *Values; + // Offset into OptTable's SubCommandIDsTable. + unsigned SubCommandIDsOffset; bool hasNoPrefix() const { return PrefixesOffset == 0; } @@ -94,6 +103,21 @@ public: getNumPrefixes(PrefixesTable)); } + bool hasSubCommands() const { return SubCommandIDsOffset != 0; } + + unsigned getNumSubCommandIDs(ArrayRef<unsigned> SubCommandIDsTable) const { + // We embed the number of subcommand IDs in the value of the first offset. + return SubCommandIDsTable[SubCommandIDsOffset]; + } + + ArrayRef<unsigned> + getSubCommandIDs(ArrayRef<unsigned> SubCommandIDsTable) const { + return hasSubCommands() ? SubCommandIDsTable.slice( + SubCommandIDsOffset + 1, + getNumSubCommandIDs(SubCommandIDsTable)) + : ArrayRef<unsigned>(); + } + void appendPrefixes(const StringTable &StrTable, ArrayRef<StringTable::Offset> PrefixesTable, SmallVectorImpl<StringRef> &Prefixes) const { @@ -119,6 +143,22 @@ public: } }; +public: + bool isValidForSubCommand(const Info *CandidateInfo, + StringRef SubCommand) const { + assert(!SubCommand.empty() && + "This helper is only for valid registered subcommands."); + auto SCIT = + std::find_if(SubCommands.begin(), SubCommands.end(), + [&](const auto &C) { return SubCommand == C.Name; }); + assert(SCIT != SubCommands.end() && + "This helper is only for valid registered subcommands."); + auto SubCommandIDs = CandidateInfo->getSubCommandIDs(SubCommandIDsTable); + unsigned CurrentSubCommandID = SCIT - &SubCommands[0]; + return std::find(SubCommandIDs.begin(), SubCommandIDs.end(), + CurrentSubCommandID) != SubCommandIDs.end(); + } + private: // A unified string table for these options. Individual strings are stored as // null terminated C-strings at offsets within this table. @@ -134,6 +174,13 @@ private: ArrayRef<Info> OptionInfos; bool IgnoreCase; + + /// The subcommand information table. + ArrayRef<SubCommand> SubCommands; + + /// The subcommand IDs table. + ArrayRef<unsigned> SubCommandIDsTable; + bool GroupedShortOptions = false; bool DashDashParsing = false; const char *EnvVar = nullptr; @@ -168,7 +215,9 @@ protected: /// manually call \c buildPrefixChars once they are fully constructed. OptTable(const StringTable &StrTable, ArrayRef<StringTable::Offset> PrefixesTable, - ArrayRef<Info> OptionInfos, bool IgnoreCase = false); + ArrayRef<Info> OptionInfos, bool IgnoreCase = false, + ArrayRef<SubCommand> SubCommands = {}, + ArrayRef<unsigned> SubCommandIDsTable = {}); /// Build (or rebuild) the PrefixChars member. void buildPrefixChars(); @@ -179,6 +228,8 @@ public: /// Return the string table used for option names. const StringTable &getStrTable() const { return *StrTable; } + ArrayRef<SubCommand> getSubCommands() const { return SubCommands; } + /// Return the prefixes table used for option names. ArrayRef<StringTable::Offset> getPrefixesTable() const { return PrefixesTable; @@ -410,7 +461,8 @@ public: /// texts. void printHelp(raw_ostream &OS, const char *Usage, const char *Title, bool ShowHidden = false, bool ShowAllAliases = false, - Visibility VisibilityMask = Visibility()) const; + Visibility VisibilityMask = Visibility(), + StringRef SubCommand = {}) const; void printHelp(raw_ostream &OS, const char *Usage, const char *Title, unsigned FlagsToInclude, unsigned FlagsToExclude, @@ -418,7 +470,8 @@ public: private: void internalPrintHelp(raw_ostream &OS, const char *Usage, const char *Title, - bool ShowHidden, bool ShowAllAliases, + StringRef SubCommand, bool ShowHidden, + bool ShowAllAliases, std::function<bool(const Info &)> ExcludeOption, Visibility VisibilityMask) const; }; @@ -428,7 +481,9 @@ class GenericOptTable : public OptTable { protected: LLVM_ABI GenericOptTable(const StringTable &StrTable, ArrayRef<StringTable::Offset> PrefixesTable, - ArrayRef<Info> OptionInfos, bool IgnoreCase = false); + ArrayRef<Info> OptionInfos, bool IgnoreCase = false, + ArrayRef<SubCommand> SubCommands = {}, + ArrayRef<unsigned> SubCommandIDsTable = {}); }; class PrecomputedOptTable : public OptTable { @@ -437,8 +492,11 @@ protected: ArrayRef<StringTable::Offset> PrefixesTable, ArrayRef<Info> OptionInfos, ArrayRef<StringTable::Offset> PrefixesUnionOffsets, - bool IgnoreCase = false) - : OptTable(StrTable, PrefixesTable, OptionInfos, IgnoreCase) { + bool IgnoreCase = false, + ArrayRef<SubCommand> SubCommands = {}, + ArrayRef<unsigned> SubCommandIDsTable = {}) + : OptTable(StrTable, PrefixesTable, OptionInfos, IgnoreCase, SubCommands, + SubCommandIDsTable) { for (auto PrefixOffset : PrefixesUnionOffsets) PrefixesUnion.push_back(StrTable[PrefixOffset]); buildPrefixChars(); @@ -452,33 +510,36 @@ protected: #define LLVM_MAKE_OPT_ID_WITH_ID_PREFIX( \ ID_PREFIX, PREFIXES_OFFSET, PREFIXED_NAME_OFFSET, ID, KIND, GROUP, ALIAS, \ ALIASARGS, FLAGS, VISIBILITY, PARAM, HELPTEXT, HELPTEXTSFORVARIANTS, \ - METAVAR, VALUES) \ + METAVAR, VALUES, SUBCOMMANDIDS_OFFSET) \ ID_PREFIX##ID #define LLVM_MAKE_OPT_ID(PREFIXES_OFFSET, PREFIXED_NAME_OFFSET, ID, KIND, \ GROUP, ALIAS, ALIASARGS, FLAGS, VISIBILITY, PARAM, \ - HELPTEXT, HELPTEXTSFORVARIANTS, METAVAR, VALUES) \ - LLVM_MAKE_OPT_ID_WITH_ID_PREFIX(OPT_, PREFIXES_OFFSET, PREFIXED_NAME_OFFSET, \ - ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, \ - VISIBILITY, PARAM, HELPTEXT, \ - HELPTEXTSFORVARIANTS, METAVAR, VALUES) + HELPTEXT, HELPTEXTSFORVARIANTS, METAVAR, VALUES, \ + SUBCOMMANDIDS_OFFSET) \ + LLVM_MAKE_OPT_ID_WITH_ID_PREFIX( \ + OPT_, PREFIXES_OFFSET, PREFIXED_NAME_OFFSET, ID, KIND, GROUP, ALIAS, \ + ALIASARGS, FLAGS, VISIBILITY, PARAM, HELPTEXT, HELPTEXTSFORVARIANTS, \ + METAVAR, VALUES, SUBCOMMANDIDS_OFFSET) #define LLVM_CONSTRUCT_OPT_INFO_WITH_ID_PREFIX( \ ID_PREFIX, PREFIXES_OFFSET, PREFIXED_NAME_OFFSET, ID, KIND, GROUP, ALIAS, \ ALIASARGS, FLAGS, VISIBILITY, PARAM, HELPTEXT, HELPTEXTSFORVARIANTS, \ - METAVAR, VALUES) \ + METAVAR, VALUES, SUBCOMMANDIDS_OFFSET) \ llvm::opt::OptTable::Info { \ PREFIXES_OFFSET, PREFIXED_NAME_OFFSET, HELPTEXT, HELPTEXTSFORVARIANTS, \ METAVAR, ID_PREFIX##ID, llvm::opt::Option::KIND##Class, PARAM, FLAGS, \ - VISIBILITY, ID_PREFIX##GROUP, ID_PREFIX##ALIAS, ALIASARGS, VALUES \ + VISIBILITY, ID_PREFIX##GROUP, ID_PREFIX##ALIAS, ALIASARGS, VALUES, \ + SUBCOMMANDIDS_OFFSET \ } #define LLVM_CONSTRUCT_OPT_INFO( \ PREFIXES_OFFSET, PREFIXED_NAME_OFFSET, ID, KIND, GROUP, ALIAS, ALIASARGS, \ - FLAGS, VISIBILITY, PARAM, HELPTEXT, HELPTEXTSFORVARIANTS, METAVAR, VALUES) \ + FLAGS, VISIBILITY, PARAM, HELPTEXT, HELPTEXTSFORVARIANTS, METAVAR, VALUES, \ + SUBCOMMANDIDS_OFFSET) \ LLVM_CONSTRUCT_OPT_INFO_WITH_ID_PREFIX( \ OPT_, PREFIXES_OFFSET, PREFIXED_NAME_OFFSET, ID, KIND, GROUP, ALIAS, \ ALIASARGS, FLAGS, VISIBILITY, PARAM, HELPTEXT, HELPTEXTSFORVARIANTS, \ - METAVAR, VALUES) + METAVAR, VALUES, SUBCOMMANDIDS_OFFSET) #endif // LLVM_OPTION_OPTTABLE_H diff --git a/llvm/include/llvm/Option/Option.h b/llvm/include/llvm/Option/Option.h index 51c330a..192cb3c9 100644 --- a/llvm/include/llvm/Option/Option.h +++ b/llvm/include/llvm/Option/Option.h @@ -216,6 +216,12 @@ public: /// always be false. LLVM_ABI bool matches(OptSpecifier ID) const; + LLVM_ABI bool isRegisteredSC(StringRef SubCommand) const { + assert(Info && "Must have a valid info!"); + assert(Owner && "Must have a valid owner!"); + return Owner->isValidForSubCommand(Info, SubCommand); + } + /// Potentially accept the current argument, returning a new Arg instance, /// or 0 if the option does not accept this argument (or the argument is /// missing values). diff --git a/llvm/include/llvm/Support/GlobPattern.h b/llvm/include/llvm/Support/GlobPattern.h index 62ed4a0..c1b4484 100644 --- a/llvm/include/llvm/Support/GlobPattern.h +++ b/llvm/include/llvm/Support/GlobPattern.h @@ -65,13 +65,19 @@ public: bool isTrivialMatchAll() const { if (!Prefix.empty()) return false; + if (!Suffix.empty()) + return false; if (SubGlobs.size() != 1) return false; return SubGlobs[0].getPat() == "*"; } + StringRef prefix() const { return Prefix; } + StringRef suffix() const { return Suffix; } + private: StringRef Prefix; + StringRef Suffix; struct SubGlobPattern { /// \param Pat the pattern to match against diff --git a/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h b/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h index 754714d..eaca0a8 100644 --- a/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h +++ b/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h @@ -79,6 +79,19 @@ public: LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; +/// Additional 'norecurse' attribute deduction during postlink LTO phase. +/// +/// This is a module pass that infers 'norecurse' attribute on functions. +/// It runs during LTO and analyzes the module's call graph to find functions +/// that are guaranteed not to call themselves, either directly or indirectly. +/// The pass uses a module-wide flag which checks if any function's address is +/// taken or any function in the module has external linkage, to safely handle +/// indirect and library function calls from current function. +class NoRecurseLTOInferencePass + : public PassInfoMixin<NoRecurseLTOInferencePass> { +public: + LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); +}; } // end namespace llvm #endif // LLVM_TRANSFORMS_IPO_FUNCTIONATTRS_H |