diff options
Diffstat (limited to 'llvm/include')
21 files changed, 1241 insertions, 340 deletions
diff --git a/llvm/include/llvm/ADT/iterator_range.h b/llvm/include/llvm/ADT/iterator_range.h index 2dc2279..7d288ea 100644 --- a/llvm/include/llvm/ADT/iterator_range.h +++ b/llvm/include/llvm/ADT/iterator_range.h @@ -43,8 +43,8 @@ class iterator_range { IteratorT begin_iterator, end_iterator; public: -#if __GNUC__ == 7 - // Be careful no to break gcc-7 on the mlir target. +#if __GNUC__ == 7 || (__GNUC__ == 8 && __GNUC_MINOR__ < 4) + // Be careful no to break gcc-7 and gcc-8 < 8.4 on the mlir target. // See https://github.com/llvm/llvm-project/issues/63843 template <typename Container> #else diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h index a0ad7d7..7713c55 100644 --- a/llvm/include/llvm/CodeGen/MachinePassManager.h +++ b/llvm/include/llvm/CodeGen/MachinePassManager.h @@ -25,17 +25,18 @@ #include "llvm/ADT/FunctionExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/IR/PassManager.h" +#include "llvm/IR/PassManagerInternal.h" #include "llvm/Support/Error.h" -#include <map> - namespace llvm { class Module; class Function; class MachineFunction; extern template class AnalysisManager<MachineFunction>; +using MachineFunctionAnalysisManager = AnalysisManager<MachineFunction>; /// A CRTP mix-in that provides informational APIs needed for machine passes. /// @@ -46,217 +47,247 @@ struct MachinePassInfoMixin : public PassInfoMixin<DerivedT> { // TODO: Add MachineFunctionProperties support. }; -/// An AnalysisManager<MachineFunction> that also exposes IR analysis results. -class MachineFunctionAnalysisManager : public AnalysisManager<MachineFunction> { -public: - using Base = AnalysisManager<MachineFunction>; +namespace detail { +struct MachinePassConcept + : PassConcept<MachineFunction, MachineFunctionAnalysisManager> { + virtual MachineFunctionProperties getRequiredProperties() const = 0; + virtual MachineFunctionProperties getSetProperties() const = 0; + virtual MachineFunctionProperties getClearedProperties() const = 0; +}; - MachineFunctionAnalysisManager() : FAM(nullptr), MAM(nullptr) {} - MachineFunctionAnalysisManager(FunctionAnalysisManager &FAM, - ModuleAnalysisManager &MAM) - : FAM(&FAM), MAM(&MAM) {} - MachineFunctionAnalysisManager(MachineFunctionAnalysisManager &&) = default; - MachineFunctionAnalysisManager & - operator=(MachineFunctionAnalysisManager &&) = default; +template <typename PassT> struct MachinePassModel : MachinePassConcept { + explicit MachinePassModel(PassT Pass) : Pass(std::move(Pass)) {} + // We have to explicitly define all the special member functions because MSVC + // refuses to generate them. + MachinePassModel(const MachinePassModel &Arg) : Pass(Arg.Pass) {} + MachinePassModel(MachinePassModel &&Arg) : Pass(std::move(Arg.Pass)) {} - /// Get the result of an analysis pass for a Function. - /// - /// Runs the analysis if a cached result is not available. - template <typename PassT> typename PassT::Result &getResult(Function &F) { - return FAM->getResult<PassT>(F); + friend void swap(MachinePassModel &LHS, MachinePassModel &RHS) { + using std::swap; + swap(LHS.Pass, RHS.Pass); } - /// Get the cached result of an analysis pass for a Function. - /// - /// This method never runs the analysis. - /// - /// \returns null if there is no cached result. - template <typename PassT> - typename PassT::Result *getCachedResult(Function &F) { - return FAM->getCachedResult<PassT>(F); + MachinePassModel &operator=(MachinePassModel RHS) { + swap(*this, RHS); + return *this; } - /// Get the result of an analysis pass for a Module. - /// - /// Runs the analysis if a cached result is not available. - template <typename PassT> typename PassT::Result &getResult(Module &M) { - return MAM->getResult<PassT>(M); + PreservedAnalyses run(MachineFunction &IR, + MachineFunctionAnalysisManager &AM) override { + return Pass.run(IR, AM); } - /// Get the cached result of an analysis pass for a Module. - /// - /// This method never runs the analysis. - /// - /// \returns null if there is no cached result. - template <typename PassT> typename PassT::Result *getCachedResult(Module &M) { - return MAM->getCachedResult<PassT>(M); + void printPipeline( + raw_ostream &OS, + function_ref<StringRef(StringRef)> MapClassName2PassName) override { + Pass.printPipeline(OS, MapClassName2PassName); } - /// Get the result of an analysis pass for a MachineFunction. - /// - /// Runs the analysis if a cached result is not available. - using Base::getResult; + StringRef name() const override { return PassT::name(); } - /// Get the cached result of an analysis pass for a MachineFunction. - /// - /// This method never runs the analysis. - /// - /// returns null if there is no cached result. - using Base::getCachedResult; - - // FIXME: Add LoopAnalysisManager or CGSCCAnalysisManager if needed. - FunctionAnalysisManager *FAM; - ModuleAnalysisManager *MAM; -}; + template <typename T> + using has_required_t = decltype(std::declval<T &>().isRequired()); + template <typename T> + static std::enable_if_t<is_detected<has_required_t, T>::value, bool> + passIsRequiredImpl() { + return T::isRequired(); + } + template <typename T> + static std::enable_if_t<!is_detected<has_required_t, T>::value, bool> + passIsRequiredImpl() { + return false; + } + bool isRequired() const override { return passIsRequiredImpl<PassT>(); } + + template <typename T> + using has_get_required_properties_t = + decltype(std::declval<T &>().getRequiredProperties()); + template <typename T> + static std::enable_if_t<is_detected<has_get_required_properties_t, T>::value, + MachineFunctionProperties> + getRequiredPropertiesImpl() { + return PassT::getRequiredProperties(); + } + template <typename T> + static std::enable_if_t<!is_detected<has_get_required_properties_t, T>::value, + MachineFunctionProperties> + getRequiredPropertiesImpl() { + return MachineFunctionProperties(); + } + MachineFunctionProperties getRequiredProperties() const override { + return getRequiredPropertiesImpl<PassT>(); + } -extern template class PassManager<MachineFunction>; + template <typename T> + using has_get_set_properties_t = + decltype(std::declval<T &>().getSetProperties()); + template <typename T> + static std::enable_if_t<is_detected<has_get_set_properties_t, T>::value, + MachineFunctionProperties> + getSetPropertiesImpl() { + return PassT::getSetProperties(); + } + template <typename T> + static std::enable_if_t<!is_detected<has_get_set_properties_t, T>::value, + MachineFunctionProperties> + getSetPropertiesImpl() { + return MachineFunctionProperties(); + } + MachineFunctionProperties getSetProperties() const override { + return getSetPropertiesImpl<PassT>(); + } -/// MachineFunctionPassManager adds/removes below features to/from the base -/// PassManager template instantiation. -/// -/// - Support passes that implement doInitialization/doFinalization. This is for -/// machine function passes to work on module level constructs. One such pass -/// is AsmPrinter. -/// -/// - Support machine module pass which runs over the module (for example, -/// MachineOutliner). A machine module pass needs to define the method: -/// -/// ```Error run(Module &, MachineFunctionAnalysisManager &)``` -/// -/// FIXME: machine module passes still need to define the usual machine -/// function pass interface, namely, -/// `PreservedAnalyses run(MachineFunction &, -/// MachineFunctionAnalysisManager &)` -/// But this interface wouldn't be executed. It is just a placeholder -/// to satisfy the pass manager type-erased inteface. This -/// special-casing of machine module pass is due to its limited use -/// cases and the unnecessary complexity it may bring to the machine -/// pass manager. -/// -/// - The base class `run` method is replaced by an alternative `run` method. -/// See details below. -/// -/// - Support codegening in the SCC order. Users include interprocedural -/// register allocation (IPRA). -class MachineFunctionPassManager - : public PassManager<MachineFunction, MachineFunctionAnalysisManager> { - using Base = PassManager<MachineFunction, MachineFunctionAnalysisManager>; + template <typename T> + using has_get_cleared_properties_t = + decltype(std::declval<T &>().getClearedProperties()); + template <typename T> + static std::enable_if_t<is_detected<has_get_cleared_properties_t, T>::value, + MachineFunctionProperties> + getClearedPropertiesImpl() { + return PassT::getClearedProperties(); + } + template <typename T> + static std::enable_if_t<!is_detected<has_get_cleared_properties_t, T>::value, + MachineFunctionProperties> + getClearedPropertiesImpl() { + return MachineFunctionProperties(); + } + MachineFunctionProperties getClearedProperties() const override { + return getClearedPropertiesImpl<PassT>(); + } + PassT Pass; +}; +} // namespace detail + +using MachineFunctionAnalysisManagerModuleProxy = + InnerAnalysisManagerProxy<MachineFunctionAnalysisManager, Module>; + +template <> +bool MachineFunctionAnalysisManagerModuleProxy::Result::invalidate( + Module &M, const PreservedAnalyses &PA, + ModuleAnalysisManager::Invalidator &Inv); +extern template class InnerAnalysisManagerProxy<MachineFunctionAnalysisManager, + Module>; + +extern template class OuterAnalysisManagerProxy<ModuleAnalysisManager, + MachineFunction>; +/// Provide the \c ModuleAnalysisManager to \c Function proxy. +using ModuleAnalysisManagerMachineFunctionProxy = + OuterAnalysisManagerProxy<ModuleAnalysisManager, MachineFunction>; + +class FunctionAnalysisManagerMachineFunctionProxy + : public AnalysisInfoMixin<FunctionAnalysisManagerMachineFunctionProxy> { public: - MachineFunctionPassManager(bool RequireCodeGenSCCOrder = false, - bool VerifyMachineFunction = false) - : RequireCodeGenSCCOrder(RequireCodeGenSCCOrder), - VerifyMachineFunction(VerifyMachineFunction) {} - MachineFunctionPassManager(MachineFunctionPassManager &&) = default; - MachineFunctionPassManager & - operator=(MachineFunctionPassManager &&) = default; - - /// Run machine passes for a Module. + class Result { + public: + explicit Result(FunctionAnalysisManager &FAM) : FAM(&FAM) {} + + Result(Result &&Arg) : FAM(std::move(Arg.FAM)) { + // We have to null out the analysis manager in the moved-from state + // because we are taking ownership of the responsibilty to clear the + // analysis state. + Arg.FAM = nullptr; + } + + ~Result() { + // FAM is cleared in a moved from state where there is nothing to do. + if (!FAM) + return; + + // Clear out the analysis manager if we're being destroyed -- it means we + // didn't even see an invalidate call when we got invalidated. + FAM->clear(); + } + + Result &operator=(Result &&RHS) { + FAM = RHS.FAM; + // We have to null out the analysis manager in the moved-from state + // because we are taking ownership of the responsibilty to clear the + // analysis state. + RHS.FAM = nullptr; + return *this; + } + + /// Accessor for the analysis manager. + FunctionAnalysisManager &getManager() { return *FAM; } + + /// Handler for invalidation of the outer IR unit, \c IRUnitT. + /// + /// If the proxy analysis itself is not preserved, we assume that the set of + /// inner IR objects contained in IRUnit may have changed. In this case, + /// we have to call \c clear() on the inner analysis manager, as it may now + /// have stale pointers to its inner IR objects. + /// + /// Regardless of whether the proxy analysis is marked as preserved, all of + /// the analyses in the inner analysis manager are potentially invalidated + /// based on the set of preserved analyses. + bool invalidate(MachineFunction &IR, const PreservedAnalyses &PA, + MachineFunctionAnalysisManager::Invalidator &Inv); + + private: + FunctionAnalysisManager *FAM; + }; + + explicit FunctionAnalysisManagerMachineFunctionProxy( + FunctionAnalysisManager &FAM) + : FAM(&FAM) {} + + /// Run the analysis pass and create our proxy result object. /// - /// The intended use is to start the codegen pipeline for a Module. The base - /// class's `run` method is deliberately hidden by this due to the observation - /// that we don't yet have the use cases of compositing two instances of - /// machine pass managers, or compositing machine pass managers with other - /// types of pass managers. - Error run(Module &M, MachineFunctionAnalysisManager &MFAM); - - template <typename PassT> void addPass(PassT &&Pass) { - Base::addPass(std::forward<PassT>(Pass)); - PassConceptT *P = Passes.back().get(); - addDoInitialization<PassT>(P); - addDoFinalization<PassT>(P); - - // Add machine module pass. - addRunOnModule<PassT>(P); + /// This doesn't do any interesting work; it is primarily used to insert our + /// proxy result object into the outer analysis cache so that we can proxy + /// invalidation to the inner analysis manager. + Result run(MachineFunction &, MachineFunctionAnalysisManager &) { + return Result(*FAM); } -private: - template <typename PassT> - using has_init_t = decltype(std::declval<PassT &>().doInitialization( - std::declval<Module &>(), - std::declval<MachineFunctionAnalysisManager &>())); - - template <typename PassT> - std::enable_if_t<!is_detected<has_init_t, PassT>::value> - addDoInitialization(PassConceptT *Pass) {} - - template <typename PassT> - std::enable_if_t<is_detected<has_init_t, PassT>::value> - addDoInitialization(PassConceptT *Pass) { - using PassModelT = detail::PassModel<MachineFunction, PassT, - MachineFunctionAnalysisManager>; - auto *P = static_cast<PassModelT *>(Pass); - InitializationFuncs.emplace_back( - [=](Module &M, MachineFunctionAnalysisManager &MFAM) { - return P->Pass.doInitialization(M, MFAM); - }); - } + static AnalysisKey Key; - template <typename PassT> - using has_fini_t = decltype(std::declval<PassT &>().doFinalization( - std::declval<Module &>(), - std::declval<MachineFunctionAnalysisManager &>())); - - template <typename PassT> - std::enable_if_t<!is_detected<has_fini_t, PassT>::value> - addDoFinalization(PassConceptT *Pass) {} - - template <typename PassT> - std::enable_if_t<is_detected<has_fini_t, PassT>::value> - addDoFinalization(PassConceptT *Pass) { - using PassModelT = detail::PassModel<MachineFunction, PassT, - MachineFunctionAnalysisManager>; - auto *P = static_cast<PassModelT *>(Pass); - FinalizationFuncs.emplace_back( - [=](Module &M, MachineFunctionAnalysisManager &MFAM) { - return P->Pass.doFinalization(M, MFAM); - }); - } +private: + FunctionAnalysisManager *FAM; +}; - template <typename PassT> - using is_machine_module_pass_t = decltype(std::declval<PassT &>().run( - std::declval<Module &>(), - std::declval<MachineFunctionAnalysisManager &>())); - - template <typename PassT> - using is_machine_function_pass_t = decltype(std::declval<PassT &>().run( - std::declval<MachineFunction &>(), - std::declval<MachineFunctionAnalysisManager &>())); - - template <typename PassT> - std::enable_if_t<!is_detected<is_machine_module_pass_t, PassT>::value> - addRunOnModule(PassConceptT *Pass) {} - - template <typename PassT> - std::enable_if_t<is_detected<is_machine_module_pass_t, PassT>::value> - addRunOnModule(PassConceptT *Pass) { - static_assert(is_detected<is_machine_function_pass_t, PassT>::value, - "machine module pass needs to define machine function pass " - "api. sorry."); - - using PassModelT = detail::PassModel<MachineFunction, PassT, - MachineFunctionAnalysisManager>; - auto *P = static_cast<PassModelT *>(Pass); - MachineModulePasses.emplace( - Passes.size() - 1, - [=](Module &M, MachineFunctionAnalysisManager &MFAM) { - return P->Pass.run(M, MFAM); - }); - } +class ModuleToMachineFunctionPassAdaptor + : public PassInfoMixin<ModuleToMachineFunctionPassAdaptor> { + using MachinePassConcept = detail::MachinePassConcept; - using FuncTy = Error(Module &, MachineFunctionAnalysisManager &); - SmallVector<llvm::unique_function<FuncTy>, 4> InitializationFuncs; - SmallVector<llvm::unique_function<FuncTy>, 4> FinalizationFuncs; +public: + explicit ModuleToMachineFunctionPassAdaptor( + std::unique_ptr<MachinePassConcept> Pass) + : Pass(std::move(Pass)) {} - using PassIndex = decltype(Passes)::size_type; - std::map<PassIndex, llvm::unique_function<FuncTy>> MachineModulePasses; + /// Runs the function pass across every function in the module. + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + void printPipeline(raw_ostream &OS, + function_ref<StringRef(StringRef)> MapClassName2PassName); - // Run codegen in the SCC order. - bool RequireCodeGenSCCOrder; + static bool isRequired() { return true; } - bool VerifyMachineFunction; +private: + std::unique_ptr<MachinePassConcept> Pass; }; +template <typename MachineFunctionPassT> +ModuleToMachineFunctionPassAdaptor +createModuleToMachineFunctionPassAdaptor(MachineFunctionPassT &&Pass) { + using PassModelT = detail::MachinePassModel<MachineFunctionPassT>; + // Do not use make_unique, it causes too many template instantiations, + // causing terrible compile times. + return ModuleToMachineFunctionPassAdaptor( + std::unique_ptr<detail::MachinePassConcept>( + new PassModelT(std::forward<MachineFunctionPassT>(Pass)))); +} + +template <> +PreservedAnalyses +PassManager<MachineFunction>::run(MachineFunction &, + AnalysisManager<MachineFunction> &); +extern template class PassManager<MachineFunction>; + +/// Convenience typedef for a pass manager over functions. +using MachineFunctionPassManager = PassManager<MachineFunction>; + } // end namespace llvm #endif // LLVM_CODEGEN_MACHINEPASSMANAGER_H diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h new file mode 100644 index 0000000..412bf42 --- /dev/null +++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h @@ -0,0 +1,694 @@ +//==--------------- llvm/CodeGen/SDPatternMatch.h ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// Contains matchers for matching SelectionDAG nodes and values. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_SDPATTERNMATCH_H +#define LLVM_CODEGEN_SDPATTERNMATCH_H + +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" +#include "llvm/CodeGen/TargetLowering.h" + +namespace llvm { +namespace SDPatternMatch { + +/// MatchContext can repurpose existing patterns to behave differently under +/// a certain context. For instance, `m_Opc(ISD::ADD)` matches plain ADD nodes +/// in normal circumstances, but matches VP_ADD nodes under a custom +/// VPMatchContext. This design is meant to facilitate code / pattern reusing. +class BasicMatchContext { + const SelectionDAG *DAG; + const TargetLowering *TLI; + +public: + explicit BasicMatchContext(const SelectionDAG *DAG) + : DAG(DAG), TLI(DAG ? &DAG->getTargetLoweringInfo() : nullptr) {} + + explicit BasicMatchContext(const TargetLowering *TLI) + : DAG(nullptr), TLI(TLI) {} + + // A valid MatchContext has to implement the following functions. + + const SelectionDAG *getDAG() const { return DAG; } + + const TargetLowering *getTLI() const { return TLI; } + + /// Return true if N effectively has opcode Opcode. + bool match(SDValue N, unsigned Opcode) const { + return N->getOpcode() == Opcode; + } +}; + +template <typename Pattern, typename MatchContext> +[[nodiscard]] bool sd_context_match(SDValue N, const MatchContext &Ctx, + Pattern &&P) { + return P.match(Ctx, N); +} + +template <typename Pattern, typename MatchContext> +[[nodiscard]] bool sd_context_match(SDNode *N, const MatchContext &Ctx, + Pattern &&P) { + return sd_context_match(SDValue(N, 0), Ctx, P); +} + +template <typename Pattern> +[[nodiscard]] bool sd_match(SDNode *N, const SelectionDAG *DAG, Pattern &&P) { + return sd_context_match(N, BasicMatchContext(DAG), P); +} + +template <typename Pattern> +[[nodiscard]] bool sd_match(SDValue N, const SelectionDAG *DAG, Pattern &&P) { + return sd_context_match(N, BasicMatchContext(DAG), P); +} + +template <typename Pattern> +[[nodiscard]] bool sd_match(SDNode *N, Pattern &&P) { + return sd_match(N, nullptr, P); +} + +template <typename Pattern> +[[nodiscard]] bool sd_match(SDValue N, Pattern &&P) { + return sd_match(N, nullptr, P); +} + +// === Utilities === +struct Value_match { + SDValue MatchVal; + + Value_match() = default; + + explicit Value_match(SDValue Match) : MatchVal(Match) {} + + template <typename MatchContext> bool match(const MatchContext &, SDValue N) { + if (MatchVal) + return MatchVal == N; + return N.getNode(); + } +}; + +/// Match any valid SDValue. +inline Value_match m_Value() { return Value_match(); } + +inline Value_match m_Specific(SDValue N) { + assert(N); + return Value_match(N); +} + +struct DeferredValue_match { + SDValue &MatchVal; + + explicit DeferredValue_match(SDValue &Match) : MatchVal(Match) {} + + template <typename MatchContext> bool match(const MatchContext &, SDValue N) { + return N == MatchVal; + } +}; + +/// Similar to m_Specific, but the specific value to match is determined by +/// another sub-pattern in the same sd_match() expression. For instance, +/// We cannot match `(add V, V)` with `m_Add(m_Value(X), m_Specific(X))` since +/// `X` is not initialized at the time it got copied into `m_Specific`. Instead, +/// we should use `m_Add(m_Value(X), m_Deferred(X))`. +inline DeferredValue_match m_Deferred(SDValue &V) { + return DeferredValue_match(V); +} + +struct Opcode_match { + unsigned Opcode; + + explicit Opcode_match(unsigned Opc) : Opcode(Opc) {} + + template <typename MatchContext> + bool match(const MatchContext &Ctx, SDValue N) { + return Ctx.match(N, Opcode); + } +}; + +inline Opcode_match m_Opc(unsigned Opcode) { return Opcode_match(Opcode); } + +template <unsigned NumUses, typename Pattern> struct NUses_match { + Pattern P; + + explicit NUses_match(const Pattern &P) : P(P) {} + + template <typename MatchContext> + bool match(const MatchContext &Ctx, SDValue N) { + // SDNode::hasNUsesOfValue is pretty expensive when the SDNode produces + // multiple results, hence we check the subsequent pattern here before + // checking the number of value users. + return P.match(Ctx, N) && N->hasNUsesOfValue(NumUses, N.getResNo()); + } +}; + +template <typename Pattern> +inline NUses_match<1, Pattern> m_OneUse(const Pattern &P) { + return NUses_match<1, Pattern>(P); +} +template <unsigned N, typename Pattern> +inline NUses_match<N, Pattern> m_NUses(const Pattern &P) { + return NUses_match<N, Pattern>(P); +} + +inline NUses_match<1, Value_match> m_OneUse() { + return NUses_match<1, Value_match>(m_Value()); +} +template <unsigned N> inline NUses_match<N, Value_match> m_NUses() { + return NUses_match<N, Value_match>(m_Value()); +} + +struct Value_bind { + SDValue &BindVal; + + explicit Value_bind(SDValue &N) : BindVal(N) {} + + template <typename MatchContext> bool match(const MatchContext &, SDValue N) { + BindVal = N; + return true; + } +}; + +inline Value_bind m_Value(SDValue &N) { return Value_bind(N); } + +template <typename Pattern, typename PredFuncT> struct TLI_pred_match { + Pattern P; + PredFuncT PredFunc; + + TLI_pred_match(const PredFuncT &Pred, const Pattern &P) + : P(P), PredFunc(Pred) {} + + template <typename MatchContext> + bool match(const MatchContext &Ctx, SDValue N) { + assert(Ctx.getTLI() && "TargetLowering is required for this pattern."); + return PredFunc(*Ctx.getTLI(), N) && P.match(Ctx, N); + } +}; + +// Explicit deduction guide. +template <typename PredFuncT, typename Pattern> +TLI_pred_match(const PredFuncT &Pred, const Pattern &P) + -> TLI_pred_match<Pattern, PredFuncT>; + +/// Match legal SDNodes based on the information provided by TargetLowering. +template <typename Pattern> inline auto m_LegalOp(const Pattern &P) { + return TLI_pred_match{[](const TargetLowering &TLI, SDValue N) { + return TLI.isOperationLegal(N->getOpcode(), + N.getValueType()); + }, + P}; +} + +/// Switch to a different MatchContext for subsequent patterns. +template <typename NewMatchContext, typename Pattern> struct SwitchContext { + const NewMatchContext &Ctx; + Pattern P; + + template <typename OrigMatchContext> + bool match(const OrigMatchContext &, SDValue N) { + return P.match(Ctx, N); + } +}; + +template <typename MatchContext, typename Pattern> +inline SwitchContext<MatchContext, Pattern> m_Context(const MatchContext &Ctx, + Pattern &&P) { + return SwitchContext<MatchContext, Pattern>{Ctx, std::move(P)}; +} + +// === Value type === +struct ValueType_bind { + EVT &BindVT; + + explicit ValueType_bind(EVT &Bind) : BindVT(Bind) {} + + template <typename MatchContext> bool match(const MatchContext &, SDValue N) { + BindVT = N.getValueType(); + return true; + } +}; + +/// Retreive the ValueType of the current SDValue. +inline ValueType_bind m_VT(EVT &VT) { return ValueType_bind(VT); } + +template <typename Pattern, typename PredFuncT> struct ValueType_match { + PredFuncT PredFunc; + Pattern P; + + ValueType_match(const PredFuncT &Pred, const Pattern &P) + : PredFunc(Pred), P(P) {} + + template <typename MatchContext> + bool match(const MatchContext &Ctx, SDValue N) { + return PredFunc(N.getValueType()) && P.match(Ctx, N); + } +}; + +// Explicit deduction guide. +template <typename PredFuncT, typename Pattern> +ValueType_match(const PredFuncT &Pred, const Pattern &P) + -> ValueType_match<Pattern, PredFuncT>; + +/// Match a specific ValueType. +template <typename Pattern> +inline auto m_SpecificVT(EVT RefVT, const Pattern &P) { + return ValueType_match{[=](EVT VT) { return VT == RefVT; }, P}; +} +inline auto m_SpecificVT(EVT RefVT) { + return ValueType_match{[=](EVT VT) { return VT == RefVT; }, m_Value()}; +} + +inline auto m_Glue() { return m_SpecificVT(MVT::Glue); } +inline auto m_OtherVT() { return m_SpecificVT(MVT::Other); } + +/// Match any integer ValueTypes. +template <typename Pattern> inline auto m_IntegerVT(const Pattern &P) { + return ValueType_match{[](EVT VT) { return VT.isInteger(); }, P}; +} +inline auto m_IntegerVT() { + return ValueType_match{[](EVT VT) { return VT.isInteger(); }, m_Value()}; +} + +/// Match any floating point ValueTypes. +template <typename Pattern> inline auto m_FloatingPointVT(const Pattern &P) { + return ValueType_match{[](EVT VT) { return VT.isFloatingPoint(); }, P}; +} +inline auto m_FloatingPointVT() { + return ValueType_match{[](EVT VT) { return VT.isFloatingPoint(); }, + m_Value()}; +} + +/// Match any vector ValueTypes. +template <typename Pattern> inline auto m_VectorVT(const Pattern &P) { + return ValueType_match{[](EVT VT) { return VT.isVector(); }, P}; +} +inline auto m_VectorVT() { + return ValueType_match{[](EVT VT) { return VT.isVector(); }, m_Value()}; +} + +/// Match fixed-length vector ValueTypes. +template <typename Pattern> inline auto m_FixedVectorVT(const Pattern &P) { + return ValueType_match{[](EVT VT) { return VT.isFixedLengthVector(); }, P}; +} +inline auto m_FixedVectorVT() { + return ValueType_match{[](EVT VT) { return VT.isFixedLengthVector(); }, + m_Value()}; +} + +/// Match scalable vector ValueTypes. +template <typename Pattern> inline auto m_ScalableVectorVT(const Pattern &P) { + return ValueType_match{[](EVT VT) { return VT.isScalableVector(); }, P}; +} +inline auto m_ScalableVectorVT() { + return ValueType_match{[](EVT VT) { return VT.isScalableVector(); }, + m_Value()}; +} + +/// Match legal ValueTypes based on the information provided by TargetLowering. +template <typename Pattern> inline auto m_LegalType(const Pattern &P) { + return TLI_pred_match{[](const TargetLowering &TLI, SDValue N) { + return TLI.isTypeLegal(N.getValueType()); + }, + P}; +} + +// === Patterns combinators === +template <typename... Preds> struct And { + template <typename MatchContext> bool match(const MatchContext &, SDValue N) { + return true; + } +}; + +template <typename Pred, typename... Preds> +struct And<Pred, Preds...> : And<Preds...> { + Pred P; + And(Pred &&p, Preds &&...preds) + : And<Preds...>(std::forward<Preds>(preds)...), P(std::forward<Pred>(p)) { + } + + template <typename MatchContext> + bool match(const MatchContext &Ctx, SDValue N) { + return P.match(Ctx, N) && And<Preds...>::match(Ctx, N); + } +}; + +template <typename... Preds> struct Or { + template <typename MatchContext> bool match(const MatchContext &, SDValue N) { + return false; + } +}; + +template <typename Pred, typename... Preds> +struct Or<Pred, Preds...> : Or<Preds...> { + Pred P; + Or(Pred &&p, Preds &&...preds) + : Or<Preds...>(std::forward<Preds>(preds)...), P(std::forward<Pred>(p)) {} + + template <typename MatchContext> + bool match(const MatchContext &Ctx, SDValue N) { + return P.match(Ctx, N) || Or<Preds...>::match(Ctx, N); + } +}; + +template <typename... Preds> And<Preds...> m_AllOf(Preds &&...preds) { + return And<Preds...>(std::forward<Preds>(preds)...); +} + +template <typename... Preds> Or<Preds...> m_AnyOf(Preds &&...preds) { + return Or<Preds...>(std::forward<Preds>(preds)...); +} + +// === Generic node matching === +template <unsigned OpIdx, typename... OpndPreds> struct Operands_match { + template <typename MatchContext> + bool match(const MatchContext &Ctx, SDValue N) { + // Returns false if there are more operands than predicates; + return N->getNumOperands() == OpIdx; + } +}; + +template <unsigned OpIdx, typename OpndPred, typename... OpndPreds> +struct Operands_match<OpIdx, OpndPred, OpndPreds...> + : Operands_match<OpIdx + 1, OpndPreds...> { + OpndPred P; + + Operands_match(OpndPred &&p, OpndPreds &&...preds) + : Operands_match<OpIdx + 1, OpndPreds...>( + std::forward<OpndPreds>(preds)...), + P(std::forward<OpndPred>(p)) {} + + template <typename MatchContext> + bool match(const MatchContext &Ctx, SDValue N) { + if (OpIdx < N->getNumOperands()) + return P.match(Ctx, N->getOperand(OpIdx)) && + Operands_match<OpIdx + 1, OpndPreds...>::match(Ctx, N); + + // This is the case where there are more predicates than operands. + return false; + } +}; + +template <typename... OpndPreds> +auto m_Node(unsigned Opcode, OpndPreds &&...preds) { + return m_AllOf(m_Opc(Opcode), Operands_match<0, OpndPreds...>( + std::forward<OpndPreds>(preds)...)); +} + +/// Provide number of operands that are not chain or glue, as well as the first +/// index of such operand. +template <bool ExcludeChain> struct EffectiveOperands { + unsigned Size = 0; + unsigned FirstIndex = 0; + + explicit EffectiveOperands(SDValue N) { + const unsigned TotalNumOps = N->getNumOperands(); + FirstIndex = TotalNumOps; + for (unsigned I = 0; I < TotalNumOps; ++I) { + // Count the number of non-chain and non-glue nodes (we ignore chain + // and glue by default) and retreive the operand index offset. + EVT VT = N->getOperand(I).getValueType(); + if (VT != MVT::Glue && VT != MVT::Other) { + ++Size; + if (FirstIndex == TotalNumOps) + FirstIndex = I; + } + } + } +}; + +template <> struct EffectiveOperands<false> { + unsigned Size = 0; + unsigned FirstIndex = 0; + + explicit EffectiveOperands(SDValue N) : Size(N->getNumOperands()) {} +}; + +// === Binary operations === +template <typename LHS_P, typename RHS_P, bool Commutable = false, + bool ExcludeChain = false> +struct BinaryOpc_match { + unsigned Opcode; + LHS_P LHS; + RHS_P RHS; + + BinaryOpc_match(unsigned Opc, const LHS_P &L, const RHS_P &R) + : Opcode(Opc), LHS(L), RHS(R) {} + + template <typename MatchContext> + bool match(const MatchContext &Ctx, SDValue N) { + if (sd_context_match(N, Ctx, m_Opc(Opcode))) { + EffectiveOperands<ExcludeChain> EO(N); + assert(EO.Size == 2); + return (LHS.match(Ctx, N->getOperand(EO.FirstIndex)) && + RHS.match(Ctx, N->getOperand(EO.FirstIndex + 1))) || + (Commutable && LHS.match(Ctx, N->getOperand(EO.FirstIndex + 1)) && + RHS.match(Ctx, N->getOperand(EO.FirstIndex))); + } + + return false; + } +}; + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_BinOp(unsigned Opc, const LHS &L, + const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(Opc, L, R); +} +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, true> m_c_BinOp(unsigned Opc, const LHS &L, + const RHS &R) { + return BinaryOpc_match<LHS, RHS, true>(Opc, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false, true> +m_ChainedBinOp(unsigned Opc, const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, false, true>(Opc, L, R); +} +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, true, true> +m_c_ChainedBinOp(unsigned Opc, const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, true, true>(Opc, L, R); +} + +// Common binary operations +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, true> m_Add(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, true>(ISD::ADD, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_Sub(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(ISD::SUB, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, true> m_Mul(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, true>(ISD::MUL, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_UDiv(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(ISD::UDIV, L, R); +} +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_SDiv(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(ISD::SDIV, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_URem(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(ISD::UREM, L, R); +} +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_SRem(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(ISD::SREM, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_Shl(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(ISD::SHL, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_Sra(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(ISD::SRA, L, R); +} +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_Srl(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(ISD::SRL, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, true> m_FAdd(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, true>(ISD::FADD, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_FSub(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(ISD::FSUB, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, true> m_FMul(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, true>(ISD::FMUL, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_FDiv(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(ISD::FDIV, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_FRem(const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(ISD::FREM, L, R); +} + +// === Unary operations === +template <typename Opnd_P, bool ExcludeChain = false> struct UnaryOpc_match { + unsigned Opcode; + Opnd_P Opnd; + + UnaryOpc_match(unsigned Opc, const Opnd_P &Op) : Opcode(Opc), Opnd(Op) {} + + template <typename MatchContext> + bool match(const MatchContext &Ctx, SDValue N) { + if (sd_context_match(N, Ctx, m_Opc(Opcode))) { + EffectiveOperands<ExcludeChain> EO(N); + assert(EO.Size == 1); + return Opnd.match(Ctx, N->getOperand(EO.FirstIndex)); + } + + return false; + } +}; + +template <typename Opnd> +inline UnaryOpc_match<Opnd> m_UnaryOp(unsigned Opc, const Opnd &Op) { + return UnaryOpc_match<Opnd>(Opc, Op); +} +template <typename Opnd> +inline UnaryOpc_match<Opnd, true> m_ChainedUnaryOp(unsigned Opc, + const Opnd &Op) { + return UnaryOpc_match<Opnd, true>(Opc, Op); +} + +template <typename Opnd> inline UnaryOpc_match<Opnd> m_ZExt(const Opnd &Op) { + return UnaryOpc_match<Opnd>(ISD::ZERO_EXTEND, Op); +} + +template <typename Opnd> inline UnaryOpc_match<Opnd> m_SExt(const Opnd &Op) { + return UnaryOpc_match<Opnd>(ISD::SIGN_EXTEND, Op); +} + +template <typename Opnd> inline UnaryOpc_match<Opnd> m_AnyExt(const Opnd &Op) { + return UnaryOpc_match<Opnd>(ISD::ANY_EXTEND, Op); +} + +template <typename Opnd> inline UnaryOpc_match<Opnd> m_Trunc(const Opnd &Op) { + return UnaryOpc_match<Opnd>(ISD::TRUNCATE, Op); +} + +// === Constants === +struct ConstantInt_match { + APInt *BindVal; + + explicit ConstantInt_match(APInt *V) : BindVal(V) {} + + template <typename MatchContext> bool match(const MatchContext &, SDValue N) { + // The logics here are similar to that in + // SelectionDAG::isConstantIntBuildVectorOrConstantInt, but the latter also + // treats GlobalAddressSDNode as a constant, which is difficult to turn into + // APInt. + if (auto *C = dyn_cast_or_null<ConstantSDNode>(N.getNode())) { + if (BindVal) + *BindVal = C->getAPIntValue(); + return true; + } + + APInt Discard; + return ISD::isConstantSplatVector(N.getNode(), + BindVal ? *BindVal : Discard); + } +}; +/// Match any interger constants or splat of an integer constant. +inline ConstantInt_match m_ConstInt() { return ConstantInt_match(nullptr); } +/// Match any interger constants or splat of an integer constant; return the +/// specific constant or constant splat value. +inline ConstantInt_match m_ConstInt(APInt &V) { return ConstantInt_match(&V); } + +struct SpecificInt_match { + APInt IntVal; + + explicit SpecificInt_match(APInt APV) : IntVal(std::move(APV)) {} + + template <typename MatchContext> + bool match(const MatchContext &Ctx, SDValue N) { + APInt ConstInt; + if (sd_context_match(N, Ctx, m_ConstInt(ConstInt))) + return APInt::isSameValue(IntVal, ConstInt); + return false; + } +}; + +/// Match a specific integer constant or constant splat value. +inline SpecificInt_match m_SpecificInt(APInt V) { + return SpecificInt_match(std::move(V)); +} +inline SpecificInt_match m_SpecificInt(uint64_t V) { + return SpecificInt_match(APInt(64, V)); +} + +inline SpecificInt_match m_Zero() { return m_SpecificInt(0U); } +inline SpecificInt_match m_AllOnes() { return m_SpecificInt(~0U); } + +/// Match true boolean value based on the information provided by +/// TargetLowering. +inline auto m_True() { + return TLI_pred_match{ + [](const TargetLowering &TLI, SDValue N) { + APInt ConstVal; + if (sd_match(N, m_ConstInt(ConstVal))) + switch (TLI.getBooleanContents(N.getValueType())) { + case TargetLowering::ZeroOrOneBooleanContent: + return ConstVal.isOne(); + case TargetLowering::ZeroOrNegativeOneBooleanContent: + return ConstVal.isAllOnes(); + case TargetLowering::UndefinedBooleanContent: + return (ConstVal & 0x01) == 1; + } + + return false; + }, + m_Value()}; +} +/// Match false boolean value based on the information provided by +/// TargetLowering. +inline auto m_False() { + return TLI_pred_match{ + [](const TargetLowering &TLI, SDValue N) { + APInt ConstVal; + if (sd_match(N, m_ConstInt(ConstVal))) + switch (TLI.getBooleanContents(N.getValueType())) { + case TargetLowering::ZeroOrOneBooleanContent: + case TargetLowering::ZeroOrNegativeOneBooleanContent: + return ConstVal.isZero(); + case TargetLowering::UndefinedBooleanContent: + return (ConstVal & 0x01) == 0; + } + + return false; + }, + m_Value()}; +} +} // namespace SDPatternMatch +} // namespace llvm +#endif diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 7bb12d8..2fc1cea 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1479,46 +1479,11 @@ public: SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, - SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, - Align Alignment, MachineMemOperand::Flags MMOFlags, - const AAMDNodes &AAInfo, - const MDNode *Ranges = nullptr, - bool IsExpanding = false); - inline SDValue getStridedLoadVP( - ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, - SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, - SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, - MaybeAlign Alignment = MaybeAlign(), - MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, - const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr, - bool IsExpanding = false) { - // Ensures that codegen never sees a None Alignment. - return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride, - Mask, EVL, PtrInfo, MemVT, - Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, - AAInfo, Ranges, IsExpanding); - } - SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, - EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, - SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding = false); SDValue getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, - MachinePointerInfo PtrInfo, MaybeAlign Alignment, - MachineMemOperand::Flags MMOFlags, - const AAMDNodes &AAInfo, - const MDNode *Ranges = nullptr, - bool IsExpanding = false); - SDValue getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, - SDValue Stride, SDValue Mask, SDValue EVL, MachineMemOperand *MMO, bool IsExpanding = false); - SDValue - getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, - SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, - SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, - MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, - const AAMDNodes &AAInfo, bool IsExpanding = false); SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, @@ -1534,13 +1499,6 @@ public: bool IsCompressing = false); SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride, SDValue Mask, - SDValue EVL, MachinePointerInfo PtrInfo, - EVT SVT, Align Alignment, - MachineMemOperand::Flags MMOFlags, - const AAMDNodes &AAInfo, - bool IsCompressing = false); - SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, - SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT SVT, MachineMemOperand *MMO, bool IsCompressing = false); SDValue getIndexedStridedStoreVP(SDValue OrigStore, const SDLoc &DL, diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h index a26c44b..d368c7e 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h @@ -562,6 +562,17 @@ public: uint64_t getEntryOffset() const { return EntryOffset; } }; + /// Offsets for the start of various important tables from the start of the + /// section. + struct DWARFDebugNamesOffsets { + uint64_t CUsBase; + uint64_t BucketsBase; + uint64_t HashesBase; + uint64_t StringOffsetsBase; + uint64_t EntryOffsetsBase; + uint64_t EntriesBase; + }; + /// Represents a single accelerator table within the DWARF v5 .debug_names /// section. class NameIndex { @@ -572,12 +583,7 @@ public: // Base of the whole unit and of various important tables, as offsets from // the start of the section. uint64_t Base; - uint64_t CUsBase; - uint64_t BucketsBase; - uint64_t HashesBase; - uint64_t StringOffsetsBase; - uint64_t EntryOffsetsBase; - uint64_t EntriesBase; + DWARFDebugNamesOffsets Offsets; void dumpCUs(ScopedPrinter &W) const; void dumpLocalTUs(ScopedPrinter &W) const; @@ -638,7 +644,7 @@ public: /// Returns the Entry at the relative `Offset` from the start of the Entry /// pool. Expected<Entry> getEntryAtRelativeOffset(uint64_t Offset) const { - auto OffsetFromSection = Offset + this->EntriesBase; + auto OffsetFromSection = Offset + this->Offsets.EntriesBase; return getEntry(&OffsetFromSection); } @@ -793,6 +799,12 @@ public: const NameIndex *getCUNameIndex(uint64_t CUOffset); }; +/// Calculates the starting offsets for various sections within the +/// .debug_names section. +void findDebugNamesOffsets(DWARFDebugNames::DWARFDebugNamesOffsets &Offsets, + uint64_t HdrSize, const dwarf::DwarfFormat Format, + const DWARFDebugNames::Header &Hdr); + /// If `Name` is the name of a templated function that includes template /// parameters, returns a substring of `Name` containing no template /// parameters. diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td index 1481328b..77d207f 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMP.td +++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td @@ -710,16 +710,14 @@ def OMP_Requires : Directive<"requires"> { } def OMP_Nothing : Directive<"nothing"> {} def OMP_TargetData : Directive<"target data"> { - let allowedClauses = [ - VersionedClause<OMPC_UseDevicePtr>, - VersionedClause<OMPC_UseDeviceAddr, 50> - ]; let allowedOnceClauses = [ VersionedClause<OMPC_Device>, VersionedClause<OMPC_If> ]; let requiredClauses = [ - VersionedClause<OMPC_Map> + VersionedClause<OMPC_Map>, + VersionedClause<OMPC_UseDevicePtr>, + VersionedClause<OMPC_UseDeviceAddr, 50> ]; } def OMP_TargetEnterData : Directive<"target enter data"> { diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td index 864f87f..d22eb76 100644 --- a/llvm/include/llvm/IR/Attributes.td +++ b/llvm/include/llvm/IR/Attributes.td @@ -339,14 +339,26 @@ def UseSampleProfile : StrBoolAttr<"use-sample-profile">; def DenormalFPMath : ComplexStrAttr<"denormal-fp-math", [FnAttr]>; def DenormalFPMathF32 : ComplexStrAttr<"denormal-fp-math-f32", [FnAttr]>; +// Attribute compatiblity rules are generated to check the attribute of the +// caller and callee and decide whether inlining should be allowed. CompatRule +// and child classes are used for the rule generation. CompatRule takes only a +// compare function which could be templated with the attribute type. +// CompatRuleStrAttr takes the compare function and the string attribute for +// checking compatibility for inline substitution. class CompatRule<string F> { - // The name of the function called to check the attribute of the caller and - // callee and decide whether inlining should be allowed. The function's - // signature must match "bool(const Function&, const Function &)", where the - // first parameter is the reference to the caller and the second parameter is - // the reference to the callee. It must return false if the attributes of the - // caller and callee are incompatible, and true otherwise. + // The function's signature must match "bool(const Function&, const + // Function&)", where the first parameter is the reference to the caller and + // the second parameter is the reference to the callee. It must return false + // if the attributes of the caller and callee are incompatible, and true + // otherwise. string CompatFunc = F; + string AttrName = ""; +} + +class CompatRuleStrAttr<string F, string Attr> : CompatRule<F> { + // The checker function is extended with an third argument as the function + // attribute string "bool(const Function&, const Function&, const StringRef&)". + string AttrName = Attr; } def : CompatRule<"isEqual<SanitizeAddressAttr>">; @@ -359,7 +371,9 @@ def : CompatRule<"isEqual<ShadowCallStackAttr>">; def : CompatRule<"isEqual<UseSampleProfileAttr>">; def : CompatRule<"isEqual<NoProfileAttr>">; def : CompatRule<"checkDenormMode">; - +def : CompatRuleStrAttr<"isEqual", "sign-return-address">; +def : CompatRuleStrAttr<"isEqual", "sign-return-address-key">; +def : CompatRuleStrAttr<"isEqual", "branch-protection-pauth-lr">; class MergeRule<string F> { // The name of the function called to merge the attributes of the caller and diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h index b5dcc7f..c0ac9a4 100644 --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -78,13 +78,20 @@ public: /// Class for constant integers. class ConstantInt final : public ConstantData { friend class Constant; + friend class ConstantVector; APInt Val; - ConstantInt(IntegerType *Ty, const APInt &V); + ConstantInt(Type *Ty, const APInt &V); void destroyConstantImpl(); + /// Return a ConstantInt with the specified value and an implied Type. The + /// type is the vector type whose integer element type corresponds to the bit + /// width of the value. + static ConstantInt *get(LLVMContext &Context, ElementCount EC, + const APInt &V); + public: ConstantInt(const ConstantInt &) = delete; @@ -136,7 +143,7 @@ public: /// Return the constant's value. inline const APInt &getValue() const { return Val; } - /// getBitWidth - Return the bitwidth of this constant. + /// getBitWidth - Return the scalar bitwidth of this constant. unsigned getBitWidth() const { return Val.getBitWidth(); } /// Return the constant as a 64-bit unsigned integer value after it @@ -259,6 +266,7 @@ public: /// class ConstantFP final : public ConstantData { friend class Constant; + friend class ConstantVector; APFloat Val; @@ -266,6 +274,12 @@ class ConstantFP final : public ConstantData { void destroyConstantImpl(); + /// Return a ConstantFP with the specified value and an implied Type. The + /// type is the vector type whose element type has the same floating point + /// semantics as the value. + static ConstantFP *get(LLVMContext &Context, ElementCount EC, + const APFloat &V); + public: ConstantFP(const ConstantFP &) = delete; diff --git a/llvm/include/llvm/IR/DebugProgramInstruction.h b/llvm/include/llvm/IR/DebugProgramInstruction.h index 1fa6b6f..9708909 100644 --- a/llvm/include/llvm/IR/DebugProgramInstruction.h +++ b/llvm/include/llvm/IR/DebugProgramInstruction.h @@ -62,6 +62,8 @@ class BasicBlock; class MDNode; class Module; class DbgVariableIntrinsic; +class DbgInfoIntrinsic; +class DbgLabelInst; class DIAssignID; class DPMarker; class DPValue; @@ -79,14 +81,14 @@ class raw_ostream; /// deleteRecord /// clone /// isIdenticalToWhenDefined -/// isEquivalentTo /// both print methods +/// createDebugIntrinsic class DbgRecord : public ilist_node<DbgRecord> { public: /// Marker that this DbgRecord is linked into. DPMarker *Marker = nullptr; /// Subclass discriminator. - enum Kind : uint8_t { ValueKind }; + enum Kind : uint8_t { ValueKind, LabelKind }; protected: DebugLoc DbgLoc; @@ -104,9 +106,16 @@ public: void print(raw_ostream &O, bool IsForDebug = false) const; void print(raw_ostream &O, ModuleSlotTracker &MST, bool IsForDebug) const; bool isIdenticalToWhenDefined(const DbgRecord &R) const; - bool isEquivalentTo(const DbgRecord &R) const; + /// Convert this DbgRecord back into an appropriate llvm.dbg.* intrinsic. + /// \p InsertBefore Optional position to insert this intrinsic. + /// \returns A new llvm.dbg.* intrinsic representiung this DbgRecord. + DbgInfoIntrinsic *createDebugIntrinsic(Module *M, + Instruction *InsertBefore) const; ///@} + /// Same as isIdenticalToWhenDefined but checks DebugLoc too. + bool isEquivalentTo(const DbgRecord &R) const; + Kind getRecordKind() const { return RecordKind; } void setMarker(DPMarker *M) { Marker = M; } @@ -156,6 +165,38 @@ protected: ~DbgRecord() = default; }; +inline raw_ostream &operator<<(raw_ostream &OS, const DbgRecord &R) { + R.print(OS); + return OS; +} + +/// Records a position in IR for a source label (DILabel). Corresponds to the +/// llvm.dbg.label intrinsic. +/// FIXME: Rename DbgLabelRecord when DPValue is renamed to DbgVariableRecord. +class DPLabel : public DbgRecord { + DILabel *Label; + +public: + DPLabel(DILabel *Label, DebugLoc DL) + : DbgRecord(LabelKind, DL), Label(Label) { + assert(Label && "Unexpected nullptr"); + } + + DPLabel *clone() const; + void print(raw_ostream &O, bool IsForDebug = false) const; + void print(raw_ostream &ROS, ModuleSlotTracker &MST, bool IsForDebug) const; + DbgLabelInst *createDebugIntrinsic(Module *M, + Instruction *InsertBefore) const; + + void setLabel(DILabel *NewLabel) { Label = NewLabel; } + DILabel *getLabel() const { return Label; } + + /// Support type inquiry through isa, cast, and dyn_cast. + static bool classof(const DbgRecord *E) { + return E->getRecordKind() == LabelKind; + } +}; + /// Record of a variable value-assignment, aka a non instruction representation /// of the dbg.value intrinsic. /// @@ -510,11 +551,6 @@ inline raw_ostream &operator<<(raw_ostream &OS, const DPMarker &Marker) { return OS; } -inline raw_ostream &operator<<(raw_ostream &OS, const DPValue &Value) { - Value.print(OS); - return OS; -} - /// Inline helper to return a range of DPValues attached to a marker. It needs /// to be inlined as it's frequently called, but also come after the declaration /// of DPMarker. Thus: it's pre-declared by users like Instruction, then an diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index b8d578d..fbaaef8 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -531,6 +531,9 @@ public: class DbgLabelInst : public DbgInfoIntrinsic { public: DILabel *getLabel() const { return cast<DILabel>(getRawLabel()); } + void setLabel(DILabel *NewLabel) { + setArgOperand(0, MetadataAsValue::get(getContext(), NewLabel)); + } Metadata *getRawLabel() const { return cast<MetadataAsValue>(getArgOperand(0))->getMetadata(); diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index 8c0d4d5..d7c1ce1 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1933,6 +1933,26 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in { LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i32_ty]>; + def int_vp_sadd_sat : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_uadd_sat : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_ssub_sat : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; + def int_vp_usub_sat : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty]>; // Floating-point arithmetic def int_vp_fadd : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td index 921e5b95..6b045e4 100644 --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -1367,6 +1367,27 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; + + class SVE2_1VectorArg_Pred_Intrinsic + : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [llvm_anyvector_ty], + [IntrNoMem]>; + + class SVE2_1VectorArgIndexed_Pred_Intrinsic + : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [llvm_anyvector_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<ArgIndex<1>>]>; + + class SVE2_Pred_1VectorArgIndexed_Intrinsic + : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i32_ty], + [IntrNoMem, ImmArg<ArgIndex<2>>]>; + + class SVE2_Pred_1VectorArg_Intrinsic + : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [IntrNoMem]>; // NOTE: There is no relationship between these intrinsics beyond an attempt // to reuse currently identical class definitions. @@ -3610,23 +3631,10 @@ def int_aarch64_sve_extq : AdvSIMD_2VectorArgIndexed_Intrinsic; // // SVE2.1 - Move predicate to/from vector // -def int_aarch64_sve_pmov_to_pred_lane : - DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], - [llvm_anyvector_ty, llvm_i32_ty], - [IntrNoMem, ImmArg<ArgIndex<1>>]>; - -def int_aarch64_sve_pmov_to_pred_lane_zero : - DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], - [llvm_anyvector_ty], - [IntrNoMem]>; - -def int_aarch64_sve_pmov_to_vector_lane_merging : - DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i32_ty], - [IntrNoMem, ImmArg<ArgIndex<2>>]>; +def int_aarch64_sve_pmov_to_pred_lane : SVE2_1VectorArgIndexed_Pred_Intrinsic; + +def int_aarch64_sve_pmov_to_pred_lane_zero : SVE2_1VectorArg_Pred_Intrinsic; -def int_aarch64_sve_pmov_to_vector_lane_zeroing : - DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], - [IntrNoMem]>; +def int_aarch64_sve_pmov_to_vector_lane_merging : SVE2_Pred_1VectorArgIndexed_Intrinsic; + +def int_aarch64_sve_pmov_to_vector_lane_zeroing : SVE2_Pred_1VectorArg_Intrinsic;
\ No newline at end of file diff --git a/llvm/include/llvm/IR/VPIntrinsics.def b/llvm/include/llvm/IR/VPIntrinsics.def index 3b32b60..4089acf 100644 --- a/llvm/include/llvm/IR/VPIntrinsics.def +++ b/llvm/include/llvm/IR/VPIntrinsics.def @@ -293,6 +293,30 @@ BEGIN_REGISTER_VP(vp_fshr, 3, 4, VP_FSHR, -1) VP_PROPERTY_FUNCTIONAL_INTRINSIC(fshr) VP_PROPERTY_FUNCTIONAL_SDOPC(FSHR) END_REGISTER_VP(vp_fshr, VP_FSHR) + +// llvm.vp.sadd.sat(x,y,mask,vlen) +BEGIN_REGISTER_VP(vp_sadd_sat, 2, 3, VP_SADDSAT, -1) +VP_PROPERTY_FUNCTIONAL_INTRINSIC(sadd_sat) +VP_PROPERTY_FUNCTIONAL_SDOPC(SADDSAT) +END_REGISTER_VP(vp_sadd_sat, VP_SADDSAT) + +// llvm.vp.uadd.sat(x,y,mask,vlen) +BEGIN_REGISTER_VP(vp_uadd_sat, 2, 3, VP_UADDSAT, -1) +VP_PROPERTY_FUNCTIONAL_INTRINSIC(uadd_sat) +VP_PROPERTY_FUNCTIONAL_SDOPC(UADDSAT) +END_REGISTER_VP(vp_uadd_sat, VP_UADDSAT) + +// llvm.vp.ssub.sat(x,y,mask,vlen) +BEGIN_REGISTER_VP(vp_ssub_sat, 2, 3, VP_SSUBSAT, -1) +VP_PROPERTY_FUNCTIONAL_INTRINSIC(ssub_sat) +VP_PROPERTY_FUNCTIONAL_SDOPC(SSUBSAT) +END_REGISTER_VP(vp_ssub_sat, VP_SSUBSAT) + +// llvm.vp.usub.sat(x,y,mask,vlen) +BEGIN_REGISTER_VP(vp_usub_sat, 2, 3, VP_USUBSAT, -1) +VP_PROPERTY_FUNCTIONAL_INTRINSIC(usub_sat) +VP_PROPERTY_FUNCTIONAL_SDOPC(USUBSAT) +END_REGISTER_VP(vp_usub_sat, VP_USUBSAT) ///// } Integer Arithmetic ///// Floating-Point Arithmetic { diff --git a/llvm/include/llvm/LTO/Config.h b/llvm/include/llvm/LTO/Config.h index 6fb55f1..482b6e5 100644 --- a/llvm/include/llvm/LTO/Config.h +++ b/llvm/include/llvm/LTO/Config.h @@ -60,9 +60,6 @@ struct Config { bool VerifyEach = false; bool DisableVerify = false; - /// Use the standard optimization pipeline. - bool UseDefaultPipeline = false; - /// Flag to indicate that the optimizer should not assume builtins are present /// on the target. bool Freestanding = false; diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index 80bbfb7..dc60727 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -37,6 +37,7 @@ #include "llvm/CodeGen/InterleavedLoadCombine.h" #include "llvm/CodeGen/JMCInstrumenter.h" #include "llvm/CodeGen/LowerEmuTLS.h" +#include "llvm/CodeGen/MIRPrinter.h" #include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/PreISelIntrinsicLowering.h" #include "llvm/CodeGen/ReplaceWithVeclib.h" @@ -88,12 +89,8 @@ namespace llvm { #define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME) \ struct PASS_NAME : public MachinePassInfoMixin<PASS_NAME> { \ template <typename... Ts> PASS_NAME(Ts &&...) {} \ - Error run(Module &, MachineFunctionAnalysisManager &) { \ - return Error::success(); \ - } \ - PreservedAnalyses run(MachineFunction &, \ - MachineFunctionAnalysisManager &) { \ - llvm_unreachable("this api is to make new PM api happy"); \ + PreservedAnalyses run(Module &, ModuleAnalysisManager &) { \ + return PreservedAnalyses::all(); \ } \ }; #define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME) \ @@ -132,8 +129,8 @@ public: Opt.OptimizeRegAlloc = getOptLevel() != CodeGenOptLevel::None; } - Error buildPipeline(ModulePassManager &MPM, MachineFunctionPassManager &MFPM, - raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, + Error buildPipeline(ModulePassManager &MPM, raw_pwrite_stream &Out, + raw_pwrite_stream *DwoOut, CodeGenFileType FileType) const; PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const { @@ -149,7 +146,15 @@ protected: using is_function_pass_t = decltype(std::declval<PassT &>().run( std::declval<Function &>(), std::declval<FunctionAnalysisManager &>())); + template <typename PassT> + using is_machine_function_pass_t = decltype(std::declval<PassT &>().run( + std::declval<MachineFunction &>(), + std::declval<MachineFunctionAnalysisManager &>())); + // Function object to maintain state while adding codegen IR passes. + // TODO: add a Function -> MachineFunction adaptor and merge + // AddIRPass/AddMachinePass so we can have a function pipeline that runs both + // function passes and machine function passes. class AddIRPass { public: AddIRPass(ModulePassManager &MPM, const DerivedT &PB) : MPM(MPM), PB(PB) {} @@ -196,31 +201,47 @@ protected: // Function object to maintain state while adding codegen machine passes. class AddMachinePass { public: - AddMachinePass(MachineFunctionPassManager &PM, const DerivedT &PB) - : PM(PM), PB(PB) {} + AddMachinePass(ModulePassManager &MPM, const DerivedT &PB) + : MPM(MPM), PB(PB) {} + ~AddMachinePass() { + if (!MFPM.isEmpty()) + MPM.addPass(createModuleToMachineFunctionPassAdaptor(std::move(MFPM))); + } + + template <typename PassT> + void operator()(PassT &&Pass, bool Force = false, + StringRef Name = PassT::name()) { + static_assert((is_detected<is_machine_function_pass_t, PassT>::value || + is_detected<is_module_pass_t, PassT>::value) && + "Only module pass and function pass are supported."); - template <typename PassT> void operator()(PassT &&Pass) { - if (!PB.runBeforeAdding(PassT::name())) + if (!Force && !PB.runBeforeAdding(Name)) return; - PM.addPass(std::forward<PassT>(Pass)); + // Add Function Pass + if constexpr (is_detected<is_machine_function_pass_t, PassT>::value) { + MFPM.addPass(std::forward<PassT>(Pass)); - for (auto &C : PB.AfterCallbacks) - C(PassT::name()); - } + for (auto &C : PB.AfterCallbacks) + C(Name); + } else { + // Add Module Pass + if (!MFPM.isEmpty()) { + MPM.addPass( + createModuleToMachineFunctionPassAdaptor(std::move(MFPM))); + MFPM = MachineFunctionPassManager(); + } - template <typename PassT> void insertPass(StringRef PassName, PassT Pass) { - PB.AfterCallbacks.emplace_back( - [this, PassName, Pass = std::move(Pass)](StringRef Name) { - if (PassName == Name) - this->PM.addPass(std::move(Pass)); - }); - } + MPM.addPass(std::forward<PassT>(Pass)); - MachineFunctionPassManager releasePM() { return std::move(PM); } + for (auto &C : PB.AfterCallbacks) + C(Name); + } + } private: - MachineFunctionPassManager &PM; + ModulePassManager &MPM; + MachineFunctionPassManager MFPM; const DerivedT &PB; }; @@ -467,30 +488,43 @@ private: template <typename Derived> Error CodeGenPassBuilder<Derived>::buildPipeline( - ModulePassManager &MPM, MachineFunctionPassManager &MFPM, - raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, + ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType) const { auto StartStopInfo = TargetPassConfig::getStartStopInfo(*PIC); if (!StartStopInfo) return StartStopInfo.takeError(); setStartStopPasses(*StartStopInfo); - AddIRPass addIRPass(MPM, derived()); - // `ProfileSummaryInfo` is always valid. - addIRPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>()); - addIRPass(RequireAnalysisPass<CollectorMetadataAnalysis, Module>()); - addISelPasses(addIRPass); - AddMachinePass addPass(MFPM, derived()); + bool PrintAsm = TargetPassConfig::willCompleteCodeGenPipeline(); + bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null; + + { + AddIRPass addIRPass(MPM, derived()); + addIRPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>()); + addIRPass(RequireAnalysisPass<CollectorMetadataAnalysis, Module>()); + addISelPasses(addIRPass); + } + + AddMachinePass addPass(MPM, derived()); + + if (PrintMIR) + addPass(PrintMIRPreparePass(Out), /*Force=*/true); + if (auto Err = addCoreISelPasses(addPass)) return std::move(Err); if (auto Err = derived().addMachinePasses(addPass)) return std::move(Err); - derived().addAsmPrinter( - addPass, [this, &Out, DwoOut, FileType](MCContext &Ctx) { - return this->TM.createMCStreamer(Out, DwoOut, FileType, Ctx); - }); + if (PrintAsm) { + derived().addAsmPrinter( + addPass, [this, &Out, DwoOut, FileType](MCContext &Ctx) { + return this->TM.createMCStreamer(Out, DwoOut, FileType, Ctx); + }); + } + + if (PrintMIR) + addPass(PrintMIRPass(Out), /*Force=*/true); addPass(FreeMachineFunctionPass()); return verifyStartStop(*StartStopInfo); diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 10c5b7c..6822cfd 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -133,7 +133,8 @@ public: void crossRegisterProxies(LoopAnalysisManager &LAM, FunctionAnalysisManager &FAM, CGSCCAnalysisManager &CGAM, - ModuleAnalysisManager &MAM); + ModuleAnalysisManager &MAM, + MachineFunctionAnalysisManager *MFAM = nullptr); /// Registers all available module analysis passes. /// @@ -569,9 +570,9 @@ public: ModulePipelineParsingCallbacks.push_back(C); } void registerPipelineParsingCallback( - const std::function<bool(StringRef Name, MachineFunctionPassManager &)> - &C) { - MachinePipelineParsingCallbacks.push_back(C); + const std::function<bool(StringRef Name, MachineFunctionPassManager &, + ArrayRef<PipelineElement>)> &C) { + MachineFunctionPipelineParsingCallbacks.push_back(C); } /// @}} @@ -733,8 +734,10 @@ private: // Machine pass callbackcs SmallVector<std::function<void(MachineFunctionAnalysisManager &)>, 2> MachineFunctionAnalysisRegistrationCallbacks; - SmallVector<std::function<bool(StringRef, MachineFunctionPassManager &)>, 2> - MachinePipelineParsingCallbacks; + SmallVector<std::function<bool(StringRef, MachineFunctionPassManager &, + ArrayRef<PipelineElement>)>, + 2> + MachineFunctionPipelineParsingCallbacks; }; /// This utility template takes care of adding require<> and invalidate<> diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h index 7462f61..d7ce088 100644 --- a/llvm/include/llvm/Target/TargetMachine.h +++ b/llvm/include/llvm/Target/TargetMachine.h @@ -34,8 +34,6 @@ using ModulePassManager = PassManager<Module>; class Function; class GlobalValue; -class MachineFunctionPassManager; -class MachineFunctionAnalysisManager; class MachineModuleInfoWrapperPass; class Mangler; class MCAsmInfo; @@ -455,11 +453,9 @@ public: bool DisableVerify = true, MachineModuleInfoWrapperPass *MMIWP = nullptr) override; - virtual Error buildCodeGenPipeline(ModulePassManager &, - MachineFunctionPassManager &, - MachineFunctionAnalysisManager &, - raw_pwrite_stream &, raw_pwrite_stream *, - CodeGenFileType, CGPassBuilderOption, + virtual Error buildCodeGenPipeline(ModulePassManager &, raw_pwrite_stream &, + raw_pwrite_stream *, CodeGenFileType, + CGPassBuilderOption, PassInstrumentationCallbacks *) { return make_error<StringError>("buildCodeGenPipeline is not overridden", inconvertibleErrorCode()); diff --git a/llvm/include/llvm/Target/TargetSchedule.td b/llvm/include/llvm/Target/TargetSchedule.td index e2781a5..48c9387 100644 --- a/llvm/include/llvm/Target/TargetSchedule.td +++ b/llvm/include/llvm/Target/TargetSchedule.td @@ -399,6 +399,8 @@ def NoSchedPred : MCSchedPredicate<TruePred>; class SchedVar<SchedPredicateBase pred, list<SchedReadWrite> selected> { SchedPredicateBase Predicate = pred; list<SchedReadWrite> Selected = selected; + // SchedModel silences warnings but is ignored. + SchedMachineModel SchedModel = ?; } // SchedModel silences warnings but is ignored. diff --git a/llvm/include/llvm/TargetParser/PPCTargetParser.def b/llvm/include/llvm/TargetParser/PPCTargetParser.def index f2c44b4..88c7304 100644 --- a/llvm/include/llvm/TargetParser/PPCTargetParser.def +++ b/llvm/include/llvm/TargetParser/PPCTargetParser.def @@ -126,4 +126,61 @@ PPC_LNX_CPU("power10",47) #undef PPC_LNX_DEFINE_OFFSETS #undef PPC_LNX_FEATURE #undef PPC_LNX_CPU + +// Definition of the following values are found in the AIX header +// file: </usr/include/sys/systemcfg.h>. +#ifndef AIX_POWERPC_USE_SYS_CONF + #define AIX_POWERPC_USE_SYS_CONF + #define AIX_SYSCON_IMPL_IDX 1 + #define AIX_PPC7_VALUE 0x00008000 + #define AIX_PPC8_VALUE 0x00010000 + #define AIX_PPC9_VALUE 0x00020000 + #define AIX_PPC10_VALUE 0x00040000 + + // Supported SUPPORT_METHOD values. + #define AIX_BUILTIN_PPC_TRUE 1 + #define AIX_BUILTIN_PPC_FALSE 0 + #define USE_SYS_CONF 2 + + // Supported COMPARE_OP values. + #define COMP_EQ 0 + +#endif + +// The value of SUPPORT_METHOD can be AIX_BUILTIN_PPC_TRUE, +// AIX_BUILTIN_PPC_FALSE, or USE_SYS_CONF. +// When the value of SUPPORT_METHOD is USE_SYS_CONF, the return value +// depends on the result of comparing the data member of +// _system_configuration specified by INDEX with a certain value. + +#ifndef PPC_AIX_CPU + #define PPC_AIX_CPU(NAME, SUPPORT_METHOD, INDEX, COMPARE_OP, VALUE) +#endif + +// __builtin_cpu_is() is supported only on Power7 and up. +PPC_AIX_CPU("power4",AIX_BUILTIN_PPC_FALSE,0,0,0) +PPC_AIX_CPU("ppc970",AIX_BUILTIN_PPC_FALSE,0,0,0) +PPC_AIX_CPU("power5",AIX_BUILTIN_PPC_FALSE,0,0,0) +PPC_AIX_CPU("power5+",AIX_BUILTIN_PPC_FALSE,0,0,0) +PPC_AIX_CPU("power6",AIX_BUILTIN_PPC_FALSE,0,0,0) +PPC_AIX_CPU("ppc-cell-be",AIX_BUILTIN_PPC_FALSE,0,0,0) +PPC_AIX_CPU("power6x",AIX_BUILTIN_PPC_FALSE,0,0,0) +PPC_AIX_CPU("ppca2",AIX_BUILTIN_PPC_FALSE,0,0,0) +PPC_AIX_CPU("ppc405",AIX_BUILTIN_PPC_FALSE,0,0,0) +PPC_AIX_CPU("ppc440",AIX_BUILTIN_PPC_FALSE,0,0,0) +PPC_AIX_CPU("ppc464",AIX_BUILTIN_PPC_FALSE,0,0,0) +PPC_AIX_CPU("ppc476",AIX_BUILTIN_PPC_FALSE,0,0,0) +PPC_AIX_CPU("power7",USE_SYS_CONF,AIX_SYSCON_IMPL_IDX,COMP_EQ,AIX_PPC7_VALUE) +PPC_AIX_CPU("power8",USE_SYS_CONF,AIX_SYSCON_IMPL_IDX,COMP_EQ,AIX_PPC8_VALUE) +PPC_AIX_CPU("power9",USE_SYS_CONF,AIX_SYSCON_IMPL_IDX,COMP_EQ,AIX_PPC9_VALUE) +PPC_AIX_CPU("power10",USE_SYS_CONF,AIX_SYSCON_IMPL_IDX,COMP_EQ,AIX_PPC10_VALUE) +#undef PPC_AIX_CPU + +// PPC_SYSTEMCONFIG_TYPE defines the IR data structure of kernel variable +// `_system_configuration`, that is found in the AIX OS header file: </usr/include/sys/systemcfg.h>. +#ifndef PPC_SYSTEMCONFIG_TYPE +#define PPC_SYSTEMCONFIG_TYPE \ +Int32Ty, Int32Ty, Int32Ty +#endif + #endif // !PPC_TGT_PARSER_UNDEF_MACROS diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def index 4c630c1..a9ed56f 100644 --- a/llvm/include/llvm/TargetParser/X86TargetParser.def +++ b/llvm/include/llvm/TargetParser/X86TargetParser.def @@ -265,6 +265,7 @@ X86_MICROARCH_LEVEL(X86_64_BASELINE,"x86-64", 95) X86_MICROARCH_LEVEL(X86_64_V2, "x86-64-v2", 96) X86_MICROARCH_LEVEL(X86_64_V3, "x86-64-v3", 97) X86_MICROARCH_LEVEL(X86_64_V4, "x86-64-v4", 98) +X86_MICROARCH_LEVEL(APXF, "apxf", 111) #undef X86_FEATURE_COMPAT #undef X86_FEATURE #undef X86_MICROARCH_LEVEL diff --git a/llvm/include/llvm/Transforms/IPO/HotColdSplitting.h b/llvm/include/llvm/Transforms/IPO/HotColdSplitting.h index c87c645..13dda6d 100644 --- a/llvm/include/llvm/Transforms/IPO/HotColdSplitting.h +++ b/llvm/include/llvm/Transforms/IPO/HotColdSplitting.h @@ -24,6 +24,7 @@ class TargetTransformInfo; class OptimizationRemarkEmitter; class AssumptionCache; class DominatorTree; +class CodeExtractor; class CodeExtractorAnalysisCache; /// A sequence of basic blocks. @@ -43,19 +44,17 @@ public: private: bool isFunctionCold(const Function &F) const; - bool isBasicBlockCold(BasicBlock* BB, - BranchProbability ColdProbThresh, - SmallPtrSetImpl<BasicBlock *> &ColdBlocks, + bool isBasicBlockCold(BasicBlock *BB, BranchProbability ColdProbThresh, SmallPtrSetImpl<BasicBlock *> &AnnotatedColdBlocks, BlockFrequencyInfo *BFI) const; bool shouldOutlineFrom(const Function &F) const; bool outlineColdRegions(Function &F, bool HasProfileSummary); - Function *extractColdRegion(const BlockSequence &Region, + bool isSplittingBeneficial(CodeExtractor &CE, const BlockSequence &Region, + TargetTransformInfo &TTI); + Function *extractColdRegion(BasicBlock &EntryPoint, CodeExtractor &CE, const CodeExtractorAnalysisCache &CEAC, - DominatorTree &DT, BlockFrequencyInfo *BFI, - TargetTransformInfo &TTI, - OptimizationRemarkEmitter &ORE, - AssumptionCache *AC, unsigned Count); + BlockFrequencyInfo *BFI, TargetTransformInfo &TTI, + OptimizationRemarkEmitter &ORE); ProfileSummaryInfo *PSI; function_ref<BlockFrequencyInfo *(Function &)> GetBFI; function_ref<TargetTransformInfo &(Function &)> GetTTI; |