diff options
123 files changed, 6031 insertions, 2905 deletions
diff --git a/bolt/unittests/Core/CMakeLists.txt b/bolt/unittests/Core/CMakeLists.txt index 54e8ea1..f10b0d9 100644 --- a/bolt/unittests/Core/CMakeLists.txt +++ b/bolt/unittests/Core/CMakeLists.txt @@ -11,6 +11,11 @@ add_bolt_unittest(CoreTests MemoryMaps.cpp DynoStats.cpp + # FIXME CoreTests uses `llvm::detail::TakeError(llvm::Error)`, but linking + # to LLVMTestingSupport introduces a transitive dependency on the + # dynamic LLVM library when LLVM_LINK_LLVM_DYLIB is ON. + ${LLVM_MAIN_SRC_DIR}/lib/Testing/Support/Error.cpp + DISABLE_LLVM_LINK_LLVM_DYLIB ) @@ -20,7 +25,6 @@ target_link_libraries(CoreTests LLVMBOLTRewrite LLVMBOLTProfile LLVMBOLTUtils - LLVMTestingSupport ) foreach (tgt ${BOLT_TARGETS_TO_BUILD}) diff --git a/bolt/unittests/Profile/CMakeLists.txt b/bolt/unittests/Profile/CMakeLists.txt index ce01c6c4..7b3cbd2 100644 --- a/bolt/unittests/Profile/CMakeLists.txt +++ b/bolt/unittests/Profile/CMakeLists.txt @@ -16,7 +16,6 @@ target_link_libraries(ProfileTests LLVMBOLTCore LLVMBOLTProfile LLVMTargetParser - LLVMTestingSupport ) foreach (tgt ${BOLT_TARGETS_TO_BUILD}) diff --git a/clang/include/clang/Basic/ABIVersions.def b/clang/include/clang/Basic/ABIVersions.def new file mode 100644 index 0000000..f6524bc --- /dev/null +++ b/clang/include/clang/Basic/ABIVersions.def @@ -0,0 +1,135 @@ +//===--- ABIVersions.def - Clang ABI Versions Database ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file enumerates Clang ABI versions. +// +//===----------------------------------------------------------------------===// +// +/// @file ABIVersions.def +/// +/// In this file, each of the Clang ABI Versions is enumerated +/// ABI_VER_MAJOR_MINOR, ABI_VER_MAJOR, or ABI_VER_LATEST macro. +/// +/// ABI_VER_MAJOR is used when the minor version is 0 or can be omitted. +/// +/// The first argument of ABI_VER_MAJOR_MINOR and ABI_VER_MAJOR is the major +/// version. +/// +/// The second argument of ABI_VER_MAJOR_MINOR is the minor version. +/// +/// The first argument of ABI_VER_LATEST is an identifier `Latest`. + +#if defined(ABI_VER_MAJOR_MINOR) != defined(ABI_VER_MAJOR) || \ + defined(ABI_VER_MAJOR) != defined(ABI_VER_LATEST) +# error ABI_VER_MAJOR_MINOR, ABI_VER_MAJOR and ABI_VER_LATEST should be defined simultaneously +#endif + +#ifndef ABI_VER_MAJOR_MINOR +# define ABI_VER_MAJOR_MINOR(Major, Minor) +#endif + +#ifndef ABI_VER_MAJOR +# define ABI_VER_MAJOR(Major) +#endif + +#ifndef ABI_VER_LATEST +# define ABI_VER_LATEST(Latest) +#endif + +/// Attempt to be ABI-compatible with code generated by Clang 3.8.x +/// (SVN r257626). This causes <1 x long long> to be passed in an integer +/// register instead of an SSE register on x64_64. +ABI_VER_MAJOR_MINOR(3, 8) + +/// Attempt to be ABI-compatible with code generated by Clang 4.0.x +/// (SVN r291814). This causes move operations to be ignored when determining +/// whether a class type can be passed or returned directly. +ABI_VER_MAJOR(4) + +/// Attempt to be ABI-compatible with code generated by Clang 6.0.x +/// (SVN r321711). This causes determination of whether a type is +/// standard-layout to ignore collisions between empty base classes and between +/// base classes and member subobjects, which affects whether we reuse base +/// class tail padding in some ABIs. +ABI_VER_MAJOR(6) + +/// Attempt to be ABI-compatible with code generated by Clang 7.0.x +/// (SVN r338536). This causes alignof (C++) and _Alignof (C11) to be compatible +/// with __alignof (i.e., return the preferred alignment) rather than returning +/// the required alignment. +ABI_VER_MAJOR(7) + +/// Attempt to be ABI-compatible with code generated by Clang 9.0.x +/// (SVN r351319). This causes vectors of __int128 to be passed in memory +/// instead of passing in multiple scalar registers on x86_64 on Linux and +/// NetBSD. +ABI_VER_MAJOR(9) + +/// Attempt to be ABI-compatible with code generated by Clang 11.0.x +/// (git 2e10b7a39b93). This causes clang to pass unions with a 256-bit vector +/// member on the stack instead of using registers, to not properly mangle +/// substitutions for template names in some cases, and to mangle declaration +/// template arguments without a cast to the parameter type even when that can +/// lead to mangling collisions. +ABI_VER_MAJOR(11) + +/// Attempt to be ABI-compatible with code generated by Clang 12.0.x +/// (git 8e464dd76bef). This causes clang to mangle lambdas within global-scope +/// inline variables incorrectly. +ABI_VER_MAJOR(12) + +/// Attempt to be ABI-compatible with code generated by Clang 14.0.x. +/// This causes clang to: +/// - mangle dependent nested names incorrectly. +/// - make trivial only those defaulted copy constructors with a +/// parameter-type-list equivalent to the parameter-type-list of an implicit +/// declaration. +ABI_VER_MAJOR(14) + +/// Attempt to be ABI-compatible with code generated by Clang 15.0.x. +/// This causes clang to: +/// - Reverse the implementation for CWG692, CWG1395 and CWG1432. +/// - pack non-POD members of packed structs. +/// - consider classes with defaulted special member functions non-pod. +ABI_VER_MAJOR(15) + +/// Attempt to be ABI-compatible with code generated by Clang 17.0.x. +/// This causes clang to revert some fixes to its implementation of the Itanium +/// name mangling scheme, with the consequence that overloaded function +/// templates are mangled the same if they differ only by: +/// - constraints +/// - whether a non-type template parameter has a deduced type +/// - the parameter list of a template template parameter +ABI_VER_MAJOR(17) + +/// Attempt to be ABI-compatible with code generated by Clang 18.0.x. +/// This causes clang to revert some fixes to the mangling of lambdas in the +/// initializers of members of local classes. +ABI_VER_MAJOR(18) + +/// Attempt to be ABI-compatible with code generated by Clang 19.0.x. +/// This causes clang to: +/// - Incorrectly mangle the 'base type' substitutions of the CXX construction +/// vtable because it hasn't added 'type' as a substitution. +/// - Skip mangling enclosing class templates of member-like friend function +/// templates. +/// - Ignore empty struct arguments in C++ mode for ARM, instead of passing +/// them as if they had a size of 1 byte. +ABI_VER_MAJOR(19) + +/// Attempt to be ABI-compatible with code generated by Clang 20.0.x. +/// This causes clang to: +/// - Incorrectly return C++ records in AVX registers on x86_64. +ABI_VER_MAJOR(20) + +/// Conform to the underlying platform's C and C++ ABIs as closely as we can. +ABI_VER_LATEST(Latest) + +#undef ABI_VER_MAJOR_MINOR +#undef ABI_VER_MAJOR +#undef ABI_VER_LATEST diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h index 0407897..569584b 100644 --- a/clang/include/clang/Basic/LangOptions.h +++ b/clang/include/clang/Basic/LangOptions.h @@ -186,95 +186,10 @@ public: /// Clang versions with different platform ABI conformance. enum class ClangABI { - /// Attempt to be ABI-compatible with code generated by Clang 3.8.x - /// (SVN r257626). This causes <1 x long long> to be passed in an - /// integer register instead of an SSE register on x64_64. - Ver3_8, - - /// Attempt to be ABI-compatible with code generated by Clang 4.0.x - /// (SVN r291814). This causes move operations to be ignored when - /// determining whether a class type can be passed or returned directly. - Ver4, - - /// Attempt to be ABI-compatible with code generated by Clang 6.0.x - /// (SVN r321711). This causes determination of whether a type is - /// standard-layout to ignore collisions between empty base classes - /// and between base classes and member subobjects, which affects - /// whether we reuse base class tail padding in some ABIs. - Ver6, - - /// Attempt to be ABI-compatible with code generated by Clang 7.0.x - /// (SVN r338536). This causes alignof (C++) and _Alignof (C11) to be - /// compatible with __alignof (i.e., return the preferred alignment) - /// rather than returning the required alignment. - Ver7, - - /// Attempt to be ABI-compatible with code generated by Clang 9.0.x - /// (SVN r351319). This causes vectors of __int128 to be passed in memory - /// instead of passing in multiple scalar registers on x86_64 on Linux and - /// NetBSD. - Ver9, - - /// Attempt to be ABI-compatible with code generated by Clang 11.0.x - /// (git 2e10b7a39b93). This causes clang to pass unions with a 256-bit - /// vector member on the stack instead of using registers, to not properly - /// mangle substitutions for template names in some cases, and to mangle - /// declaration template arguments without a cast to the parameter type - /// even when that can lead to mangling collisions. - Ver11, - - /// Attempt to be ABI-compatible with code generated by Clang 12.0.x - /// (git 8e464dd76bef). This causes clang to mangle lambdas within - /// global-scope inline variables incorrectly. - Ver12, - - /// Attempt to be ABI-compatible with code generated by Clang 14.0.x. - /// This causes clang to: - /// - mangle dependent nested names incorrectly. - /// - make trivial only those defaulted copy constructors with a - /// parameter-type-list equivalent to the parameter-type-list of an - /// implicit declaration. - Ver14, - - /// Attempt to be ABI-compatible with code generated by Clang 15.0.x. - /// This causes clang to: - /// - Reverse the implementation for DR692, DR1395 and DR1432. - /// - pack non-POD members of packed structs. - /// - consider classes with defaulted special member functions non-pod. - Ver15, - - /// Attempt to be ABI-compatible with code generated by Clang 17.0.x. - /// This causes clang to revert some fixes to its implementation of the - /// Itanium name mangling scheme, with the consequence that overloaded - /// function templates are mangled the same if they differ only by: - /// - constraints - /// - whether a non-type template parameter has a deduced type - /// - the parameter list of a template template parameter - Ver17, - - /// Attempt to be ABI-compatible with code generated by Clang 18.0.x. - /// This causes clang to revert some fixes to the mangling of lambdas - /// in the initializers of members of local classes. - Ver18, - - /// Attempt to be ABI-compatible with code generated by Clang 19.0.x. - /// This causes clang to: - /// - Incorrectly mangle the 'base type' substitutions of the CXX - /// construction vtable because it hasn't added 'type' as a substitution. - /// - Skip mangling enclosing class templates of member-like friend - /// function templates. - /// - Ignore empty struct arguments in C++ mode for ARM, instead of - /// passing them as if they had a size of 1 byte. - Ver19, - - /// Attempt to be ABI-compatible with code generated by Clang 20.0.x. - /// This causes clang to: - /// - Incorrectly return C++ records in AVX registers on x86_64. - Ver20, - - /// Conform to the underlying platform's C and C++ ABIs as closely - /// as we can. - Latest +#define ABI_VER_MAJOR_MINOR(Major, Minor) Ver##Major##_##Minor, +#define ABI_VER_MAJOR(Major) Ver##Major, +#define ABI_VER_LATEST(Latest) Latest +#include "clang/Basic/ABIVersions.def" }; enum class CoreFoundationABI { diff --git a/clang/lib/AST/ByteCode/EvalEmitter.cpp b/clang/lib/AST/ByteCode/EvalEmitter.cpp index 976b7c0..9ed61c7 100644 --- a/clang/lib/AST/ByteCode/EvalEmitter.cpp +++ b/clang/lib/AST/ByteCode/EvalEmitter.cpp @@ -292,7 +292,7 @@ bool EvalEmitter::emitGetLocal(uint32_t I, const SourceInfo &Info) { Block *B = getLocal(I); - if (!CheckLocalLoad(S, OpPC, Pointer(B))) + if (!CheckLocalLoad(S, OpPC, B)) return false; S.Stk.push<T>(*reinterpret_cast<T *>(B->data())); diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp index eb4e480..6999fee 100644 --- a/clang/lib/AST/ByteCode/Interp.cpp +++ b/clang/lib/AST/ByteCode/Interp.cpp @@ -211,25 +211,26 @@ static void diagnoseNonConstVariable(InterpState &S, CodePtr OpPC, S.Note(VD->getLocation(), diag::note_declared_at); } -static bool CheckTemporary(InterpState &S, CodePtr OpPC, const Pointer &Ptr, +static bool CheckTemporary(InterpState &S, CodePtr OpPC, const Block *B, AccessKinds AK) { - if (auto ID = Ptr.getDeclID()) { - if (!Ptr.isStaticTemporary()) + if (B->getDeclID()) { + if (!(B->isStatic() && B->isTemporary())) return true; const auto *MTE = dyn_cast_if_present<MaterializeTemporaryExpr>( - Ptr.getDeclDesc()->asExpr()); + B->getDescriptor()->asExpr()); if (!MTE) return true; // FIXME(perf): Since we do this check on every Load from a static // temporary, it might make sense to cache the value of the // isUsableInConstantExpressions call. - if (!MTE->isUsableInConstantExpressions(S.getASTContext()) && - Ptr.block()->getEvalID() != S.Ctx.getEvalID()) { + if (B->getEvalID() != S.Ctx.getEvalID() && + !MTE->isUsableInConstantExpressions(S.getASTContext())) { const SourceInfo &E = S.Current->getSource(OpPC); S.FFDiag(E, diag::note_constexpr_access_static_temporary, 1) << AK; - S.Note(Ptr.getDeclLoc(), diag::note_constexpr_temporary_here); + S.Note(B->getDescriptor()->getLocation(), + diag::note_constexpr_temporary_here); return false; } } @@ -658,17 +659,19 @@ static bool CheckVolatile(InterpState &S, CodePtr OpPC, const Pointer &Ptr, return false; } -bool CheckInitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr, - AccessKinds AK) { +bool DiagnoseUninitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr, + AccessKinds AK) { assert(Ptr.isLive()); + assert(!Ptr.isInitialized()); + return DiagnoseUninitialized(S, OpPC, Ptr.isExtern(), Ptr.getDeclDesc(), AK); +} - if (Ptr.isInitialized()) - return true; - - if (Ptr.isExtern() && S.checkingPotentialConstantExpression()) +bool DiagnoseUninitialized(InterpState &S, CodePtr OpPC, bool Extern, + const Descriptor *Desc, AccessKinds AK) { + if (Extern && S.checkingPotentialConstantExpression()) return false; - if (const auto *VD = Ptr.getDeclDesc()->asVarDecl(); + if (const auto *VD = Desc->asVarDecl(); VD && (VD->isConstexpr() || VD->hasGlobalStorage())) { if (VD == S.EvaluatingDecl && @@ -703,9 +706,9 @@ bool CheckInitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr, return false; } -static bool CheckLifetime(InterpState &S, CodePtr OpPC, const Pointer &Ptr, +static bool CheckLifetime(InterpState &S, CodePtr OpPC, Lifetime LT, AccessKinds AK) { - if (Ptr.getLifetime() == Lifetime::Started) + if (LT == Lifetime::Started) return true; if (!S.checkingPotentialConstantExpression()) { @@ -715,11 +718,11 @@ static bool CheckLifetime(InterpState &S, CodePtr OpPC, const Pointer &Ptr, return false; } -static bool CheckWeak(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { - if (!Ptr.isWeak()) +static bool CheckWeak(InterpState &S, CodePtr OpPC, const Block *B) { + if (!B->isWeak()) return true; - const auto *VD = Ptr.getDeclDesc()->asVarDecl(); + const auto *VD = B->getDescriptor()->asVarDecl(); assert(VD); S.FFDiag(S.Current->getLocation(OpPC), diag::note_constexpr_var_init_weak) << VD; @@ -732,32 +735,56 @@ static bool CheckWeak(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { // ones removed that are impossible on primitive global values. // For example, since those can't be members of structs, they also can't // be mutable. -bool CheckGlobalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { - if (!CheckExtern(S, OpPC, Ptr)) +bool CheckGlobalLoad(InterpState &S, CodePtr OpPC, const Block *B) { + const auto &Desc = + *reinterpret_cast<const GlobalInlineDescriptor *>(B->rawData()); + if (!CheckExtern(S, OpPC, Pointer(const_cast<Block *>(B)))) return false; - if (!CheckConstant(S, OpPC, Ptr)) - return false; - if (!CheckDummy(S, OpPC, Ptr, AK_Read)) + if (!CheckConstant(S, OpPC, B->getDescriptor())) return false; - if (!CheckInitialized(S, OpPC, Ptr, AK_Read)) + if (!CheckDummy(S, OpPC, B, AK_Read)) return false; - if (!CheckTemporary(S, OpPC, Ptr, AK_Read)) + if (Desc.InitState != GlobalInitState::Initialized) + return DiagnoseUninitialized(S, OpPC, B->isExtern(), B->getDescriptor(), + AK_Read); + if (!CheckTemporary(S, OpPC, B, AK_Read)) return false; - if (!CheckWeak(S, OpPC, Ptr)) + if (!CheckWeak(S, OpPC, B)) return false; - if (!CheckVolatile(S, OpPC, Ptr, AK_Read)) + if (B->getDescriptor()->IsVolatile) { + if (!S.getLangOpts().CPlusPlus) + return Invalid(S, OpPC); + + const ValueDecl *D = B->getDescriptor()->asValueDecl(); + S.FFDiag(S.Current->getLocation(OpPC), + diag::note_constexpr_access_volatile_obj, 1) + << AK_Read << 1 << D; + S.Note(D->getLocation(), diag::note_constexpr_volatile_here) << 1; return false; + } return true; } // Similarly, for local loads. -bool CheckLocalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { - if (!CheckLifetime(S, OpPC, Ptr, AK_Read)) - return false; - if (!CheckInitialized(S, OpPC, Ptr, AK_Read)) - return false; - if (!CheckVolatile(S, OpPC, Ptr, AK_Read)) +bool CheckLocalLoad(InterpState &S, CodePtr OpPC, const Block *B) { + assert(!B->isExtern()); + const auto &Desc = *reinterpret_cast<const InlineDescriptor *>(B->rawData()); + if (!CheckLifetime(S, OpPC, Desc.LifeState, AK_Read)) + return false; + if (!Desc.IsInitialized) + return DiagnoseUninitialized(S, OpPC, /*Extern=*/false, B->getDescriptor(), + AK_Read); + if (B->getDescriptor()->IsVolatile) { + if (!S.getLangOpts().CPlusPlus) + return Invalid(S, OpPC); + + const ValueDecl *D = B->getDescriptor()->asValueDecl(); + S.FFDiag(S.Current->getLocation(OpPC), + diag::note_constexpr_access_volatile_obj, 1) + << AK_Read << 1 << D; + S.Note(D->getLocation(), diag::note_constexpr_volatile_here) << 1; return false; + } return true; } @@ -769,19 +796,19 @@ bool CheckLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr, return false; if (!CheckConstant(S, OpPC, Ptr)) return false; - if (!CheckDummy(S, OpPC, Ptr, AK)) + if (Ptr.isBlockPointer() && !CheckDummy(S, OpPC, Ptr.block(), AK)) return false; if (!CheckRange(S, OpPC, Ptr, AK)) return false; if (!CheckActive(S, OpPC, Ptr, AK)) return false; - if (!CheckLifetime(S, OpPC, Ptr, AK)) - return false; - if (!CheckInitialized(S, OpPC, Ptr, AK)) + if (!CheckLifetime(S, OpPC, Ptr.getLifetime(), AK)) return false; - if (!CheckTemporary(S, OpPC, Ptr, AK)) + if (!Ptr.isInitialized()) + return DiagnoseUninitialized(S, OpPC, Ptr, AK); + if (Ptr.isBlockPointer() && !CheckTemporary(S, OpPC, Ptr.block(), AK)) return false; - if (!CheckWeak(S, OpPC, Ptr)) + if (Ptr.isBlockPointer() && !CheckWeak(S, OpPC, Ptr.block())) return false; if (!CheckMutable(S, OpPC, Ptr)) return false; @@ -798,7 +825,7 @@ bool CheckFinalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { if (!CheckConstant(S, OpPC, Ptr)) return false; - if (!CheckDummy(S, OpPC, Ptr, AK_Read)) + if (Ptr.isBlockPointer() && !CheckDummy(S, OpPC, Ptr.block(), AK_Read)) return false; if (!CheckExtern(S, OpPC, Ptr)) return false; @@ -806,13 +833,13 @@ bool CheckFinalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { return false; if (!CheckActive(S, OpPC, Ptr, AK_Read)) return false; - if (!CheckLifetime(S, OpPC, Ptr, AK_Read)) - return false; - if (!CheckInitialized(S, OpPC, Ptr, AK_Read)) + if (!CheckLifetime(S, OpPC, Ptr.getLifetime(), AK_Read)) return false; - if (!CheckTemporary(S, OpPC, Ptr, AK_Read)) + if (!Ptr.isInitialized()) + return DiagnoseUninitialized(S, OpPC, Ptr, AK_Read); + if (Ptr.isBlockPointer() && !CheckTemporary(S, OpPC, Ptr.block(), AK_Read)) return false; - if (!CheckWeak(S, OpPC, Ptr)) + if (Ptr.isBlockPointer() && !CheckWeak(S, OpPC, Ptr.block())) return false; if (!CheckMutable(S, OpPC, Ptr)) return false; @@ -822,9 +849,9 @@ bool CheckFinalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { bool CheckStore(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { if (!CheckLive(S, OpPC, Ptr, AK_Assign)) return false; - if (!CheckDummy(S, OpPC, Ptr, AK_Assign)) + if (Ptr.isBlockPointer() && !CheckDummy(S, OpPC, Ptr.block(), AK_Assign)) return false; - if (!CheckLifetime(S, OpPC, Ptr, AK_Assign)) + if (!CheckLifetime(S, OpPC, Ptr.getLifetime(), AK_Assign)) return false; if (!CheckExtern(S, OpPC, Ptr)) return false; @@ -1098,12 +1125,11 @@ bool CheckDeclRef(InterpState &S, CodePtr OpPC, const DeclRefExpr *DR) { return diagnoseUnknownDecl(S, OpPC, D); } -bool CheckDummy(InterpState &S, CodePtr OpPC, const Pointer &Ptr, - AccessKinds AK) { - if (!Ptr.isDummy()) +bool CheckDummy(InterpState &S, CodePtr OpPC, const Block *B, AccessKinds AK) { + const Descriptor *Desc = B->getDescriptor(); + if (!Desc->isDummy()) return true; - const Descriptor *Desc = Ptr.getDeclDesc(); const ValueDecl *D = Desc->asValueDecl(); if (!D) return false; @@ -1426,7 +1452,7 @@ static bool checkConstructor(InterpState &S, CodePtr OpPC, const Function *Func, bool CheckDestructor(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { if (!CheckLive(S, OpPC, Ptr, AK_Destroy)) return false; - if (!CheckTemporary(S, OpPC, Ptr, AK_Destroy)) + if (!CheckTemporary(S, OpPC, Ptr.block(), AK_Destroy)) return false; if (!CheckRange(S, OpPC, Ptr, AK_Destroy)) return false; @@ -1749,7 +1775,7 @@ static void startLifetimeRecurse(const Pointer &Ptr) { bool StartLifetime(InterpState &S, CodePtr OpPC) { const auto &Ptr = S.Stk.peek<Pointer>(); - if (!CheckDummy(S, OpPC, Ptr, AK_Destroy)) + if (Ptr.isBlockPointer() && !CheckDummy(S, OpPC, Ptr.block(), AK_Destroy)) return false; startLifetimeRecurse(Ptr.narrow()); return true; @@ -1780,7 +1806,7 @@ static void endLifetimeRecurse(const Pointer &Ptr) { /// Ends the lifetime of the peek'd pointer. bool EndLifetime(InterpState &S, CodePtr OpPC) { const auto &Ptr = S.Stk.peek<Pointer>(); - if (!CheckDummy(S, OpPC, Ptr, AK_Destroy)) + if (Ptr.isBlockPointer() && !CheckDummy(S, OpPC, Ptr.block(), AK_Destroy)) return false; endLifetimeRecurse(Ptr.narrow()); return true; @@ -1789,7 +1815,7 @@ bool EndLifetime(InterpState &S, CodePtr OpPC) { /// Ends the lifetime of the pop'd pointer. bool EndLifetimePop(InterpState &S, CodePtr OpPC) { const auto &Ptr = S.Stk.pop<Pointer>(); - if (!CheckDummy(S, OpPC, Ptr, AK_Destroy)) + if (Ptr.isBlockPointer() && !CheckDummy(S, OpPC, Ptr.block(), AK_Destroy)) return false; endLifetimeRecurse(Ptr.narrow()); return true; @@ -1804,16 +1830,16 @@ bool CheckNewTypeMismatch(InterpState &S, CodePtr OpPC, const Expr *E, // Similar to CheckStore(), but with the additional CheckTemporary() call and // the AccessKinds are different. - if (!CheckTemporary(S, OpPC, Ptr, AK_Construct)) + if (!CheckTemporary(S, OpPC, Ptr.block(), AK_Construct)) return false; if (!CheckLive(S, OpPC, Ptr, AK_Construct)) return false; - if (!CheckDummy(S, OpPC, Ptr, AK_Construct)) + if (!CheckDummy(S, OpPC, Ptr.block(), AK_Construct)) return false; // CheckLifetime for this and all base pointers. for (Pointer P = Ptr;;) { - if (!CheckLifetime(S, OpPC, P, AK_Construct)) + if (!CheckLifetime(S, OpPC, P.getLifetime(), AK_Construct)) return false; if (P.isRoot()) diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h index 8a28106..5c17a14 100644 --- a/clang/lib/AST/ByteCode/Interp.h +++ b/clang/lib/AST/ByteCode/Interp.h @@ -51,8 +51,7 @@ bool CheckLive(InterpState &S, CodePtr OpPC, const Pointer &Ptr, AccessKinds AK); /// Checks if a pointer is a dummy pointer. -bool CheckDummy(InterpState &S, CodePtr OpPC, const Pointer &Ptr, - AccessKinds AK); +bool CheckDummy(InterpState &S, CodePtr OpPC, const Block *B, AccessKinds AK); /// Checks if a pointer is null. bool CheckNull(InterpState &S, CodePtr OpPC, const Pointer &Ptr, @@ -89,11 +88,14 @@ bool CheckLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr, AccessKinds AK = AK_Read); bool CheckFinalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr); -bool CheckInitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr, - AccessKinds AK); +bool DiagnoseUninitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr, + AccessKinds AK); +bool DiagnoseUninitialized(InterpState &S, CodePtr OpPC, bool Extern, + const Descriptor *Desc, AccessKinds AK); + /// Checks a direct load of a primitive value from a global or local variable. -bool CheckGlobalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr); -bool CheckLocalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr); +bool CheckGlobalLoad(InterpState &S, CodePtr OpPC, const Block *B); +bool CheckLocalLoad(InterpState &S, CodePtr OpPC, const Block *B); /// Checks if a value can be stored in a block. bool CheckStore(InterpState &S, CodePtr OpPC, const Pointer &Ptr); @@ -1351,10 +1353,10 @@ inline bool ConstFloat(InterpState &S, CodePtr OpPC, const Floating &F) { template <PrimType Name, class T = typename PrimConv<Name>::T> bool GetLocal(InterpState &S, CodePtr OpPC, uint32_t I) { - const Pointer &Ptr = S.Current->getLocalPointer(I); - if (!CheckLocalLoad(S, OpPC, Ptr)) + const Block *B = S.Current->getLocalBlock(I); + if (!CheckLocalLoad(S, OpPC, B)) return false; - S.Stk.push<T>(Ptr.deref<T>()); + S.Stk.push<T>(B->deref<T>()); return true; } @@ -1465,22 +1467,26 @@ bool SetThisField(InterpState &S, CodePtr OpPC, uint32_t I) { template <PrimType Name, class T = typename PrimConv<Name>::T> bool GetGlobal(InterpState &S, CodePtr OpPC, uint32_t I) { - const Pointer &Ptr = S.P.getPtrGlobal(I); + const Block *B = S.P.getGlobal(I); - if (!CheckGlobalLoad(S, OpPC, Ptr)) + if (!CheckGlobalLoad(S, OpPC, B)) return false; - S.Stk.push<T>(Ptr.deref<T>()); + S.Stk.push<T>(B->deref<T>()); return true; } /// Same as GetGlobal, but without the checks. template <PrimType Name, class T = typename PrimConv<Name>::T> bool GetGlobalUnchecked(InterpState &S, CodePtr OpPC, uint32_t I) { - const Pointer &Ptr = S.P.getPtrGlobal(I); - if (!CheckInitialized(S, OpPC, Ptr, AK_Read)) - return false; - S.Stk.push<T>(Ptr.deref<T>()); + const Block *B = S.P.getGlobal(I); + const auto &Desc = + *reinterpret_cast<const GlobalInlineDescriptor *>(B->rawData()); + if (Desc.InitState != GlobalInitState::Initialized) + return DiagnoseUninitialized(S, OpPC, B->isExtern(), B->getDescriptor(), + AK_Read); + + S.Stk.push<T>(B->deref<T>()); return true; } @@ -2351,8 +2357,8 @@ static inline bool IncDecPtrHelper(InterpState &S, CodePtr OpPC, static inline bool IncPtr(InterpState &S, CodePtr OpPC) { const Pointer &Ptr = S.Stk.pop<Pointer>(); - if (!CheckInitialized(S, OpPC, Ptr, AK_Increment)) - return false; + if (!Ptr.isInitialized()) + return DiagnoseUninitialized(S, OpPC, Ptr, AK_Increment); return IncDecPtrHelper<ArithOp::Add>(S, OpPC, Ptr); } @@ -2360,8 +2366,8 @@ static inline bool IncPtr(InterpState &S, CodePtr OpPC) { static inline bool DecPtr(InterpState &S, CodePtr OpPC) { const Pointer &Ptr = S.Stk.pop<Pointer>(); - if (!CheckInitialized(S, OpPC, Ptr, AK_Decrement)) - return false; + if (!Ptr.isInitialized()) + return DiagnoseUninitialized(S, OpPC, Ptr, AK_Decrement); return IncDecPtrHelper<ArithOp::Sub>(S, OpPC, Ptr); } diff --git a/clang/lib/AST/ByteCode/InterpBlock.h b/clang/lib/AST/ByteCode/InterpBlock.h index 5162223..07194d6 100644 --- a/clang/lib/AST/ByteCode/InterpBlock.h +++ b/clang/lib/AST/ByteCode/InterpBlock.h @@ -103,6 +103,10 @@ public: return reinterpret_cast<const std::byte *>(this) + sizeof(Block); } + template <typename T> T deref() const { + return *reinterpret_cast<const T *>(data()); + } + /// Invokes the constructor. void invokeCtor() { assert(!IsInitialized); diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp index f908d02..c835bd4 100644 --- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp +++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp @@ -276,7 +276,7 @@ static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC, if (!CheckLive(S, OpPC, StrPtr, AK_Read)) return false; - if (!CheckDummy(S, OpPC, StrPtr, AK_Read)) + if (!CheckDummy(S, OpPC, StrPtr.block(), AK_Read)) return false; assert(StrPtr.getFieldDesc()->isPrimitiveArray()); @@ -2232,7 +2232,7 @@ static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC, return false; if (!CheckMutable(S, OpPC, Ptr)) return false; - if (!CheckDummy(S, OpPC, Ptr, AK_Read)) + if (!CheckDummy(S, OpPC, Ptr.block(), AK_Read)) return false; } diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp index 9342192..f2eac86 100644 --- a/clang/lib/AST/ByteCode/InterpFrame.cpp +++ b/clang/lib/AST/ByteCode/InterpFrame.cpp @@ -231,6 +231,10 @@ Pointer InterpFrame::getLocalPointer(unsigned Offset) const { return Pointer(localBlock(Offset)); } +Block *InterpFrame::getLocalBlock(unsigned Offset) const { + return localBlock(Offset); +} + Pointer InterpFrame::getParamPointer(unsigned Off) { // Return the block if it was created previously. if (auto Pt = Params.find(Off); Pt != Params.end()) diff --git a/clang/lib/AST/ByteCode/InterpFrame.h b/clang/lib/AST/ByteCode/InterpFrame.h index cfebe93..4be5391 100644 --- a/clang/lib/AST/ByteCode/InterpFrame.h +++ b/clang/lib/AST/ByteCode/InterpFrame.h @@ -86,6 +86,7 @@ public: /// Returns a pointer to a local variables. Pointer getLocalPointer(unsigned Offset) const; + Block *getLocalBlock(unsigned Offset) const; /// Returns the value of an argument. template <typename T> const T &getParam(unsigned Offset) const { diff --git a/clang/lib/AST/ByteCode/InterpState.cpp b/clang/lib/AST/ByteCode/InterpState.cpp index a06b125..b3c0a67 100644 --- a/clang/lib/AST/ByteCode/InterpState.cpp +++ b/clang/lib/AST/ByteCode/InterpState.cpp @@ -76,9 +76,6 @@ bool InterpState::reportOverflow(const Expr *E, const llvm::APSInt &Value) { void InterpState::deallocate(Block *B) { assert(B); - const Descriptor *Desc = B->getDescriptor(); - assert(Desc); - // The block might have a pointer saved in a field in its data // that points to the block itself. We call the dtor first, // which will destroy all the data but leave InlineDescriptors @@ -95,7 +92,7 @@ void InterpState::deallocate(Block *B) { auto *D = new (Memory) DeadBlock(DeadBlocks, B); // Since the block doesn't hold any actual data anymore, we can just // memcpy() everything over. - std::memcpy(D->rawData(), B->rawData(), Desc->getAllocSize()); + std::memcpy(D->rawData(), B->rawData(), B->getSize()); D->B.IsInitialized = B->IsInitialized; // We moved the contents over to the DeadBlock. diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 9f77e62..ccc3154 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -3936,47 +3936,18 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts, GenerateArg(Consumer, OPT_fsanitize_ignorelist_EQ, F); switch (Opts.getClangABICompat()) { - case LangOptions::ClangABI::Ver3_8: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "3.8"); +#define ABI_VER_MAJOR_MINOR(Major, Minor) \ + case LangOptions::ClangABI::Ver##Major##_##Minor: \ + GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, #Major "." #Minor); \ break; - case LangOptions::ClangABI::Ver4: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "4.0"); +#define ABI_VER_MAJOR(Major) \ + case LangOptions::ClangABI::Ver##Major: \ + GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, #Major ".0"); \ break; - case LangOptions::ClangABI::Ver6: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "6.0"); - break; - case LangOptions::ClangABI::Ver7: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "7.0"); - break; - case LangOptions::ClangABI::Ver9: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "9.0"); - break; - case LangOptions::ClangABI::Ver11: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "11.0"); - break; - case LangOptions::ClangABI::Ver12: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "12.0"); - break; - case LangOptions::ClangABI::Ver14: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "14.0"); - break; - case LangOptions::ClangABI::Ver15: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "15.0"); - break; - case LangOptions::ClangABI::Ver17: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "17.0"); - break; - case LangOptions::ClangABI::Ver18: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "18.0"); - break; - case LangOptions::ClangABI::Ver19: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "19.0"); - break; - case LangOptions::ClangABI::Ver20: - GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "20.0"); - break; - case LangOptions::ClangABI::Latest: +#define ABI_VER_LATEST(Latest) \ + case LangOptions::ClangABI::Latest: \ break; +#include "clang/Basic/ABIVersions.def" } if (Opts.getSignReturnAddressScope() == @@ -4482,32 +4453,18 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args, !VerParts.second.getAsInteger(10, Minor) : VerParts.first.size() == Ver.size() || VerParts.second == "0")) { // Got a valid version number. - if (Major == 3 && Minor <= 8) - Opts.setClangABICompat(LangOptions::ClangABI::Ver3_8); - else if (Major <= 4) - Opts.setClangABICompat(LangOptions::ClangABI::Ver4); - else if (Major <= 6) - Opts.setClangABICompat(LangOptions::ClangABI::Ver6); - else if (Major <= 7) - Opts.setClangABICompat(LangOptions::ClangABI::Ver7); - else if (Major <= 9) - Opts.setClangABICompat(LangOptions::ClangABI::Ver9); - else if (Major <= 11) - Opts.setClangABICompat(LangOptions::ClangABI::Ver11); - else if (Major <= 12) - Opts.setClangABICompat(LangOptions::ClangABI::Ver12); - else if (Major <= 14) - Opts.setClangABICompat(LangOptions::ClangABI::Ver14); - else if (Major <= 15) - Opts.setClangABICompat(LangOptions::ClangABI::Ver15); - else if (Major <= 17) - Opts.setClangABICompat(LangOptions::ClangABI::Ver17); - else if (Major <= 18) - Opts.setClangABICompat(LangOptions::ClangABI::Ver18); - else if (Major <= 19) - Opts.setClangABICompat(LangOptions::ClangABI::Ver19); - else if (Major <= 20) - Opts.setClangABICompat(LangOptions::ClangABI::Ver20); +#define ABI_VER_MAJOR_MINOR(Major_, Minor_) \ + if (std::tie(Major, Minor) <= std::tuple(Major_, Minor_)) \ + Opts.setClangABICompat(LangOptions::ClangABI::Ver##Major_##_##Minor_); \ + else +#define ABI_VER_MAJOR(Major_) \ + if (Major <= Major_) \ + Opts.setClangABICompat(LangOptions::ClangABI::Ver##Major_); \ + else +#define ABI_VER_LATEST(Latest) \ + { /* Equivalent to latest version - do nothing */ \ + } +#include "clang/Basic/ABIVersions.def" } else if (Ver != "latest") { Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << A->getValue(); diff --git a/clang/lib/Headers/avx2intrin.h b/clang/lib/Headers/avx2intrin.h index dc9fc07..3c3a3d1 100644 --- a/clang/lib/Headers/avx2intrin.h +++ b/clang/lib/Headers/avx2intrin.h @@ -31,6 +31,14 @@ __min_vector_width__(128))) #endif +#if defined(__cplusplus) && (__cplusplus >= 201103L) +#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr +#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr +#else +#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 +#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 +#endif + /* SSE4 Multiple Packed Sums of Absolute Difference. */ /// Computes sixteen sum of absolute difference (SAD) operations on sets of /// four unsigned 8-bit integers from the 256-bit integer vectors \a X and @@ -460,7 +468,7 @@ _mm256_adds_epu16(__m256i __a, __m256i __b) /// \param __b /// A 256-bit integer vector. /// \returns A 256-bit integer vector containing the result. -static __inline__ __m256i __DEFAULT_FN_ATTRS256 +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_and_si256(__m256i __a, __m256i __b) { return (__m256i)((__v4du)__a & (__v4du)__b); @@ -478,7 +486,7 @@ _mm256_and_si256(__m256i __a, __m256i __b) /// \param __b /// A 256-bit integer vector. /// \returns A 256-bit integer vector containing the result. -static __inline__ __m256i __DEFAULT_FN_ATTRS256 +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_andnot_si256(__m256i __a, __m256i __b) { return (__m256i)(~(__v4du)__a & (__v4du)__b); @@ -1822,7 +1830,7 @@ _mm256_mul_epu32(__m256i __a, __m256i __b) /// \param __b /// A 256-bit integer vector. /// \returns A 256-bit integer vector containing the result. -static __inline__ __m256i __DEFAULT_FN_ATTRS256 +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_or_si256(__m256i __a, __m256i __b) { return (__m256i)((__v4du)__a | (__v4du)__b); @@ -2974,7 +2982,7 @@ _mm256_unpacklo_epi64(__m256i __a, __m256i __b) /// \param __b /// A 256-bit integer vector. /// \returns A 256-bit integer vector containing the result. -static __inline__ __m256i __DEFAULT_FN_ATTRS256 +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_xor_si256(__m256i __a, __m256i __b) { return (__m256i)((__v4du)__a ^ (__v4du)__b); @@ -5289,5 +5297,7 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) #undef __DEFAULT_FN_ATTRS256 #undef __DEFAULT_FN_ATTRS128 +#undef __DEFAULT_FN_ATTRS256_CONSTEXPR +#undef __DEFAULT_FN_ATTRS128_CONSTEXPR #endif /* __AVX2INTRIN_H */ diff --git a/clang/lib/Headers/avx512dqintrin.h b/clang/lib/Headers/avx512dqintrin.h index 88b48e3..62325b9 100644 --- a/clang/lib/Headers/avx512dqintrin.h +++ b/clang/lib/Headers/avx512dqintrin.h @@ -20,6 +20,14 @@ __attribute__((__always_inline__, __nodebug__, \ __target__("avx512dq,no-evex512"))) +#if defined(__cplusplus) && (__cplusplus >= 201103L) +#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr +#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr +#else +#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 +#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS +#endif + static __inline __mmask8 __DEFAULT_FN_ATTRS _knot_mask8(__mmask8 __M) { @@ -167,7 +175,7 @@ _mm512_maskz_mullo_epi64(__mmask8 __U, __m512i __A, __m512i __B) { (__v8di)_mm512_setzero_si512()); } -static __inline__ __m512d __DEFAULT_FN_ATTRS512 +static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_xor_pd(__m512d __A, __m512d __B) { return (__m512d)((__v8du)__A ^ (__v8du)__B); } @@ -186,7 +194,7 @@ _mm512_maskz_xor_pd(__mmask8 __U, __m512d __A, __m512d __B) { (__v8df)_mm512_setzero_pd()); } -static __inline__ __m512 __DEFAULT_FN_ATTRS512 +static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_xor_ps (__m512 __A, __m512 __B) { return (__m512)((__v16su)__A ^ (__v16su)__B); } @@ -205,7 +213,7 @@ _mm512_maskz_xor_ps(__mmask16 __U, __m512 __A, __m512 __B) { (__v16sf)_mm512_setzero_ps()); } -static __inline__ __m512d __DEFAULT_FN_ATTRS512 +static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_or_pd(__m512d __A, __m512d __B) { return (__m512d)((__v8du)__A | (__v8du)__B); } @@ -224,7 +232,7 @@ _mm512_maskz_or_pd(__mmask8 __U, __m512d __A, __m512d __B) { (__v8df)_mm512_setzero_pd()); } -static __inline__ __m512 __DEFAULT_FN_ATTRS512 +static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_or_ps(__m512 __A, __m512 __B) { return (__m512)((__v16su)__A | (__v16su)__B); } @@ -243,7 +251,7 @@ _mm512_maskz_or_ps(__mmask16 __U, __m512 __A, __m512 __B) { (__v16sf)_mm512_setzero_ps()); } -static __inline__ __m512d __DEFAULT_FN_ATTRS512 +static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_and_pd(__m512d __A, __m512d __B) { return (__m512d)((__v8du)__A & (__v8du)__B); } @@ -262,7 +270,7 @@ _mm512_maskz_and_pd(__mmask8 __U, __m512d __A, __m512d __B) { (__v8df)_mm512_setzero_pd()); } -static __inline__ __m512 __DEFAULT_FN_ATTRS512 +static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_and_ps(__m512 __A, __m512 __B) { return (__m512)((__v16su)__A & (__v16su)__B); } @@ -281,7 +289,7 @@ _mm512_maskz_and_ps(__mmask16 __U, __m512 __A, __m512 __B) { (__v16sf)_mm512_setzero_ps()); } -static __inline__ __m512d __DEFAULT_FN_ATTRS512 +static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_andnot_pd(__m512d __A, __m512d __B) { return (__m512d)(~(__v8du)__A & (__v8du)__B); } @@ -300,7 +308,7 @@ _mm512_maskz_andnot_pd(__mmask8 __U, __m512d __A, __m512d __B) { (__v8df)_mm512_setzero_pd()); } -static __inline__ __m512 __DEFAULT_FN_ATTRS512 +static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_andnot_ps(__m512 __A, __m512 __B) { return (__m512)(~(__v16su)__A & (__v16su)__B); } @@ -1375,5 +1383,7 @@ _mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A) #undef __DEFAULT_FN_ATTRS512 #undef __DEFAULT_FN_ATTRS +#undef __DEFAULT_FN_ATTRS512_CONSTEXPR +#undef __DEFAULT_FN_ATTRS_CONSTEXPR #endif diff --git a/clang/lib/Headers/avx512fintrin.h b/clang/lib/Headers/avx512fintrin.h index 45e7eeb..74343c3 100644 --- a/clang/lib/Headers/avx512fintrin.h +++ b/clang/lib/Headers/avx512fintrin.h @@ -645,7 +645,7 @@ _mm512_zextsi256_si512(__m256i __a) } /* Bitwise operators */ -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_and_epi32(__m512i __a, __m512i __b) { return (__m512i)((__v16su)__a & (__v16su)__b); @@ -666,7 +666,7 @@ _mm512_maskz_and_epi32(__mmask16 __k, __m512i __a, __m512i __b) __k, __a, __b); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_and_epi64(__m512i __a, __m512i __b) { return (__m512i)((__v8du)__a & (__v8du)__b); @@ -687,13 +687,13 @@ _mm512_maskz_and_epi64(__mmask8 __k, __m512i __a, __m512i __b) __k, __a, __b); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_andnot_si512 (__m512i __A, __m512i __B) { return (__m512i)(~(__v8du)__A & (__v8du)__B); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_andnot_epi32 (__m512i __A, __m512i __B) { return (__m512i)(~(__v16su)__A & (__v16su)__B); @@ -714,7 +714,7 @@ _mm512_maskz_andnot_epi32(__mmask16 __U, __m512i __A, __m512i __B) __U, __A, __B); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_andnot_epi64(__m512i __A, __m512i __B) { return (__m512i)(~(__v8du)__A & (__v8du)__B); @@ -735,7 +735,7 @@ _mm512_maskz_andnot_epi64(__mmask8 __U, __m512i __A, __m512i __B) __U, __A, __B); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_or_epi32(__m512i __a, __m512i __b) { return (__m512i)((__v16su)__a | (__v16su)__b); @@ -755,7 +755,7 @@ _mm512_maskz_or_epi32(__mmask16 __k, __m512i __a, __m512i __b) return (__m512i)_mm512_mask_or_epi32(_mm512_setzero_si512(), __k, __a, __b); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_or_epi64(__m512i __a, __m512i __b) { return (__m512i)((__v8du)__a | (__v8du)__b); @@ -775,7 +775,7 @@ _mm512_maskz_or_epi64(__mmask8 __k, __m512i __a, __m512i __b) return (__m512i)_mm512_mask_or_epi64(_mm512_setzero_si512(), __k, __a, __b); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_xor_epi32(__m512i __a, __m512i __b) { return (__m512i)((__v16su)__a ^ (__v16su)__b); @@ -795,7 +795,7 @@ _mm512_maskz_xor_epi32(__mmask16 __k, __m512i __a, __m512i __b) return (__m512i)_mm512_mask_xor_epi32(_mm512_setzero_si512(), __k, __a, __b); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_xor_epi64(__m512i __a, __m512i __b) { return (__m512i)((__v8du)__a ^ (__v8du)__b); @@ -815,19 +815,19 @@ _mm512_maskz_xor_epi64(__mmask8 __k, __m512i __a, __m512i __b) return (__m512i)_mm512_mask_xor_epi64(_mm512_setzero_si512(), __k, __a, __b); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_and_si512(__m512i __a, __m512i __b) { return (__m512i)((__v8du)__a & (__v8du)__b); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_or_si512(__m512i __a, __m512i __b) { return (__m512i)((__v8du)__a | (__v8du)__b); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_xor_si512(__m512i __a, __m512i __b) { return (__m512i)((__v8du)__a ^ (__v8du)__b); @@ -835,45 +835,38 @@ _mm512_xor_si512(__m512i __a, __m512i __b) /* Arithmetic */ -static __inline __m512d __DEFAULT_FN_ATTRS512 -_mm512_add_pd(__m512d __a, __m512d __b) -{ +static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_add_pd(__m512d __a, __m512d __b) { return (__m512d)((__v8df)__a + (__v8df)__b); } -static __inline __m512 __DEFAULT_FN_ATTRS512 -_mm512_add_ps(__m512 __a, __m512 __b) -{ +static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_add_ps(__m512 __a, __m512 __b) { return (__m512)((__v16sf)__a + (__v16sf)__b); } -static __inline __m512d __DEFAULT_FN_ATTRS512 -_mm512_mul_pd(__m512d __a, __m512d __b) -{ +static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mul_pd(__m512d __a, __m512d __b) { return (__m512d)((__v8df)__a * (__v8df)__b); } -static __inline __m512 __DEFAULT_FN_ATTRS512 -_mm512_mul_ps(__m512 __a, __m512 __b) -{ +static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mul_ps(__m512 __a, __m512 __b) { return (__m512)((__v16sf)__a * (__v16sf)__b); } -static __inline __m512d __DEFAULT_FN_ATTRS512 -_mm512_sub_pd(__m512d __a, __m512d __b) -{ +static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_sub_pd(__m512d __a, __m512d __b) { return (__m512d)((__v8df)__a - (__v8df)__b); } -static __inline __m512 __DEFAULT_FN_ATTRS512 -_mm512_sub_ps(__m512 __a, __m512 __b) -{ +static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_sub_ps(__m512 __a, __m512 __b) { return (__m512)((__v16sf)__a - (__v16sf)__b); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 -_mm512_add_epi64 (__m512i __A, __m512i __B) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_add_epi64(__m512i __A, __m512i __B) { return (__m512i) ((__v8du) __A + (__v8du) __B); } @@ -2315,9 +2308,8 @@ _mm_maskz_div_sd(__mmask8 __U,__m128d __A, __m128d __B) { (__v2df)_mm_setzero_pd(), \ (__mmask8)(U), (int)(R))) -static __inline __m512d __DEFAULT_FN_ATTRS512 -_mm512_div_pd(__m512d __a, __m512d __b) -{ +static __inline __m512d + __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_div_pd(__m512d __a, __m512d __b) { return (__m512d)((__v8df)__a/(__v8df)__b); } @@ -2335,9 +2327,8 @@ _mm512_maskz_div_pd(__mmask8 __U, __m512d __A, __m512d __B) { (__v8df)_mm512_setzero_pd()); } -static __inline __m512 __DEFAULT_FN_ATTRS512 -_mm512_div_ps(__m512 __a, __m512 __b) -{ +static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_div_ps(__m512 __a, __m512 __b) { return (__m512)((__v16sf)__a/(__v16sf)__b); } @@ -4123,9 +4114,8 @@ _mm512_cvtss_f32(__m512 __a) /* Unpack and Interleave */ -static __inline __m512d __DEFAULT_FN_ATTRS512 -_mm512_unpackhi_pd(__m512d __a, __m512d __b) -{ +static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_unpackhi_pd(__m512d __a, __m512d __b) { return (__m512d)__builtin_shufflevector((__v8df)__a, (__v8df)__b, 1, 9, 1+2, 9+2, 1+4, 9+4, 1+6, 9+6); } @@ -4146,9 +4136,8 @@ _mm512_maskz_unpackhi_pd(__mmask8 __U, __m512d __A, __m512d __B) (__v8df)_mm512_setzero_pd()); } -static __inline __m512d __DEFAULT_FN_ATTRS512 -_mm512_unpacklo_pd(__m512d __a, __m512d __b) -{ +static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_unpacklo_pd(__m512d __a, __m512d __b) { return (__m512d)__builtin_shufflevector((__v8df)__a, (__v8df)__b, 0, 8, 0+2, 8+2, 0+4, 8+4, 0+6, 8+6); } @@ -4169,9 +4158,8 @@ _mm512_maskz_unpacklo_pd (__mmask8 __U, __m512d __A, __m512d __B) (__v8df)_mm512_setzero_pd()); } -static __inline __m512 __DEFAULT_FN_ATTRS512 -_mm512_unpackhi_ps(__m512 __a, __m512 __b) -{ +static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_unpackhi_ps(__m512 __a, __m512 __b) { return (__m512)__builtin_shufflevector((__v16sf)__a, (__v16sf)__b, 2, 18, 3, 19, 2+4, 18+4, 3+4, 19+4, @@ -4195,9 +4183,8 @@ _mm512_maskz_unpackhi_ps (__mmask16 __U, __m512 __A, __m512 __B) (__v16sf)_mm512_setzero_ps()); } -static __inline __m512 __DEFAULT_FN_ATTRS512 -_mm512_unpacklo_ps(__m512 __a, __m512 __b) -{ +static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_unpacklo_ps(__m512 __a, __m512 __b) { return (__m512)__builtin_shufflevector((__v16sf)__a, (__v16sf)__b, 0, 16, 1, 17, 0+4, 16+4, 1+4, 17+4, @@ -5303,7 +5290,7 @@ _mm512_mask_store_epi64 (void *__P, __mmask8 __U, __m512i __A) (__mmask8) __U); } -static __inline__ __m512d __DEFAULT_FN_ATTRS512 +static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_movedup_pd (__m512d __A) { return (__m512d)__builtin_shufflevector((__v8df)__A, (__v8df)__A, @@ -8665,7 +8652,7 @@ _mm512_mask_testn_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B) _mm512_setzero_si512()); } -static __inline__ __m512 __DEFAULT_FN_ATTRS512 +static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_movehdup_ps (__m512 __A) { return (__m512)__builtin_shufflevector((__v16sf)__A, (__v16sf)__A, @@ -8688,7 +8675,7 @@ _mm512_maskz_movehdup_ps (__mmask16 __U, __m512 __A) (__v16sf)_mm512_setzero_ps()); } -static __inline__ __m512 __DEFAULT_FN_ATTRS512 +static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_moveldup_ps (__m512 __A) { return (__m512)__builtin_shufflevector((__v16sf)__A, (__v16sf)__A, @@ -9337,19 +9324,23 @@ _mm512_mask_abs_pd(__m512d __W, __mmask8 __K, __m512d __A) * This takes log2(n) steps where n is the number of elements in the vector. */ -static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_add_epi64(__m512i __W) { +static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_reduce_add_epi64(__m512i __W) { return __builtin_reduce_add((__v8di)__W); } -static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_mul_epi64(__m512i __W) { +static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_reduce_mul_epi64(__m512i __W) { return __builtin_reduce_mul((__v8di)__W); } -static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_and_epi64(__m512i __W) { +static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_reduce_and_epi64(__m512i __W) { return __builtin_reduce_and((__v8di)__W); } -static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_or_epi64(__m512i __W) { +static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_reduce_or_epi64(__m512i __W) { return __builtin_reduce_or((__v8di)__W); } @@ -9400,22 +9391,22 @@ _mm512_mask_reduce_mul_pd(__mmask8 __M, __m512d __W) { return __builtin_ia32_reduce_fmul_pd512(1.0, __W); } -static __inline__ int __DEFAULT_FN_ATTRS512 +static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_reduce_add_epi32(__m512i __W) { return __builtin_reduce_add((__v16si)__W); } -static __inline__ int __DEFAULT_FN_ATTRS512 +static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_reduce_mul_epi32(__m512i __W) { return __builtin_reduce_mul((__v16si)__W); } -static __inline__ int __DEFAULT_FN_ATTRS512 +static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_reduce_and_epi32(__m512i __W) { return __builtin_reduce_and((__v16si)__W); } -static __inline__ int __DEFAULT_FN_ATTRS512 +static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_reduce_or_epi32(__m512i __W) { return __builtin_reduce_or((__v16si)__W); } @@ -9466,22 +9457,22 @@ _mm512_mask_reduce_mul_ps(__mmask16 __M, __m512 __W) { return __builtin_ia32_reduce_fmul_ps512(1.0f, __W); } -static __inline__ long long __DEFAULT_FN_ATTRS512 +static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_reduce_max_epi64(__m512i __V) { return __builtin_reduce_max((__v8di)__V); } -static __inline__ unsigned long long __DEFAULT_FN_ATTRS512 +static __inline__ unsigned long long __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_reduce_max_epu64(__m512i __V) { return __builtin_reduce_max((__v8du)__V); } -static __inline__ long long __DEFAULT_FN_ATTRS512 +static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_reduce_min_epi64(__m512i __V) { return __builtin_reduce_min((__v8di)__V); } -static __inline__ unsigned long long __DEFAULT_FN_ATTRS512 +static __inline__ unsigned long long __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_reduce_min_epu64(__m512i __V) { return __builtin_reduce_min((__v8du)__V); } @@ -9509,22 +9500,22 @@ _mm512_mask_reduce_min_epu64(__mmask8 __M, __m512i __V) { __V = _mm512_mask_mov_epi64(_mm512_set1_epi64(-1LL), __M, __V); return __builtin_reduce_min((__v8du)__V); } -static __inline__ int __DEFAULT_FN_ATTRS512 +static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_reduce_max_epi32(__m512i __V) { return __builtin_reduce_max((__v16si)__V); } -static __inline__ unsigned int __DEFAULT_FN_ATTRS512 +static __inline__ unsigned int __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_reduce_max_epu32(__m512i __V) { return __builtin_reduce_max((__v16su)__V); } -static __inline__ int __DEFAULT_FN_ATTRS512 +static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_reduce_min_epi32(__m512i __V) { return __builtin_reduce_min((__v16si)__V); } -static __inline__ unsigned int __DEFAULT_FN_ATTRS512 +static __inline__ unsigned int __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_reduce_min_epu32(__m512i __V) { return __builtin_reduce_min((__v16su)__V); } diff --git a/clang/lib/Headers/avxintrin.h b/clang/lib/Headers/avxintrin.h index b9ca013..2be4f68 100644 --- a/clang/lib/Headers/avxintrin.h +++ b/clang/lib/Headers/avxintrin.h @@ -87,9 +87,8 @@ typedef __bf16 __m256bh __attribute__((__vector_size__(32), __aligned__(32))); /// A 256-bit vector of [4 x double] containing one of the source operands. /// \returns A 256-bit vector of [4 x double] containing the sums of both /// operands. -static __inline __m256d __DEFAULT_FN_ATTRS -_mm256_add_pd(__m256d __a, __m256d __b) -{ +static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR +_mm256_add_pd(__m256d __a, __m256d __b) { return (__m256d)((__v4df)__a+(__v4df)__b); } @@ -105,9 +104,8 @@ _mm256_add_pd(__m256d __a, __m256d __b) /// A 256-bit vector of [8 x float] containing one of the source operands. /// \returns A 256-bit vector of [8 x float] containing the sums of both /// operands. -static __inline __m256 __DEFAULT_FN_ATTRS -_mm256_add_ps(__m256 __a, __m256 __b) -{ +static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_add_ps(__m256 __a, + __m256 __b) { return (__m256)((__v8sf)__a+(__v8sf)__b); } @@ -123,9 +121,8 @@ _mm256_add_ps(__m256 __a, __m256 __b) /// A 256-bit vector of [4 x double] containing the subtrahend. /// \returns A 256-bit vector of [4 x double] containing the differences between /// both operands. -static __inline __m256d __DEFAULT_FN_ATTRS -_mm256_sub_pd(__m256d __a, __m256d __b) -{ +static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR +_mm256_sub_pd(__m256d __a, __m256d __b) { return (__m256d)((__v4df)__a-(__v4df)__b); } @@ -141,9 +138,8 @@ _mm256_sub_pd(__m256d __a, __m256d __b) /// A 256-bit vector of [8 x float] containing the subtrahend. /// \returns A 256-bit vector of [8 x float] containing the differences between /// both operands. -static __inline __m256 __DEFAULT_FN_ATTRS -_mm256_sub_ps(__m256 __a, __m256 __b) -{ +static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_sub_ps(__m256 __a, + __m256 __b) { return (__m256)((__v8sf)__a-(__v8sf)__b); } @@ -197,9 +193,8 @@ _mm256_addsub_ps(__m256 __a, __m256 __b) /// A 256-bit vector of [4 x double] containing the divisor. /// \returns A 256-bit vector of [4 x double] containing the quotients of both /// operands. -static __inline __m256d __DEFAULT_FN_ATTRS -_mm256_div_pd(__m256d __a, __m256d __b) -{ +static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR +_mm256_div_pd(__m256d __a, __m256d __b) { return (__m256d)((__v4df)__a/(__v4df)__b); } @@ -215,9 +210,8 @@ _mm256_div_pd(__m256d __a, __m256d __b) /// A 256-bit vector of [8 x float] containing the divisor. /// \returns A 256-bit vector of [8 x float] containing the quotients of both /// operands. -static __inline __m256 __DEFAULT_FN_ATTRS -_mm256_div_ps(__m256 __a, __m256 __b) -{ +static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_div_ps(__m256 __a, + __m256 __b) { return (__m256)((__v8sf)__a/(__v8sf)__b); } @@ -317,9 +311,8 @@ _mm256_min_ps(__m256 __a, __m256 __b) /// A 256-bit vector of [4 x double] containing one of the operands. /// \returns A 256-bit vector of [4 x double] containing the products of both /// operands. -static __inline __m256d __DEFAULT_FN_ATTRS -_mm256_mul_pd(__m256d __a, __m256d __b) -{ +static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR +_mm256_mul_pd(__m256d __a, __m256d __b) { return (__m256d)((__v4df)__a * (__v4df)__b); } @@ -335,9 +328,8 @@ _mm256_mul_pd(__m256d __a, __m256d __b) /// A 256-bit vector of [8 x float] containing one of the operands. /// \returns A 256-bit vector of [8 x float] containing the products of both /// operands. -static __inline __m256 __DEFAULT_FN_ATTRS -_mm256_mul_ps(__m256 __a, __m256 __b) -{ +static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_mul_ps(__m256 __a, + __m256 __b) { return (__m256)((__v8sf)__a * (__v8sf)__b); } @@ -555,7 +547,7 @@ _mm256_rcp_ps(__m256 __a) /// A 256-bit vector of [4 x double] containing one of the source operands. /// \returns A 256-bit vector of [4 x double] containing the bitwise AND of the /// values between both operands. -static __inline __m256d __DEFAULT_FN_ATTRS +static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_and_pd(__m256d __a, __m256d __b) { return (__m256d)((__v4du)__a & (__v4du)__b); @@ -573,7 +565,7 @@ _mm256_and_pd(__m256d __a, __m256d __b) /// A 256-bit vector of [8 x float] containing one of the source operands. /// \returns A 256-bit vector of [8 x float] containing the bitwise AND of the /// values between both operands. -static __inline __m256 __DEFAULT_FN_ATTRS +static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_and_ps(__m256 __a, __m256 __b) { return (__m256)((__v8su)__a & (__v8su)__b); @@ -594,7 +586,7 @@ _mm256_and_ps(__m256 __a, __m256 __b) /// \returns A 256-bit vector of [4 x double] containing the bitwise AND of the /// values of the second operand and the one's complement of the first /// operand. -static __inline __m256d __DEFAULT_FN_ATTRS +static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_andnot_pd(__m256d __a, __m256d __b) { return (__m256d)(~(__v4du)__a & (__v4du)__b); @@ -615,7 +607,7 @@ _mm256_andnot_pd(__m256d __a, __m256d __b) /// \returns A 256-bit vector of [8 x float] containing the bitwise AND of the /// values of the second operand and the one's complement of the first /// operand. -static __inline __m256 __DEFAULT_FN_ATTRS +static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_andnot_ps(__m256 __a, __m256 __b) { return (__m256)(~(__v8su)__a & (__v8su)__b); @@ -633,7 +625,7 @@ _mm256_andnot_ps(__m256 __a, __m256 __b) /// A 256-bit vector of [4 x double] containing one of the source operands. /// \returns A 256-bit vector of [4 x double] containing the bitwise OR of the /// values between both operands. -static __inline __m256d __DEFAULT_FN_ATTRS +static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_or_pd(__m256d __a, __m256d __b) { return (__m256d)((__v4du)__a | (__v4du)__b); @@ -651,7 +643,7 @@ _mm256_or_pd(__m256d __a, __m256d __b) /// A 256-bit vector of [8 x float] containing one of the source operands. /// \returns A 256-bit vector of [8 x float] containing the bitwise OR of the /// values between both operands. -static __inline __m256 __DEFAULT_FN_ATTRS +static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_or_ps(__m256 __a, __m256 __b) { return (__m256)((__v8su)__a | (__v8su)__b); @@ -669,7 +661,7 @@ _mm256_or_ps(__m256 __a, __m256 __b) /// A 256-bit vector of [4 x double] containing one of the source operands. /// \returns A 256-bit vector of [4 x double] containing the bitwise XOR of the /// values between both operands. -static __inline __m256d __DEFAULT_FN_ATTRS +static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_xor_pd(__m256d __a, __m256d __b) { return (__m256d)((__v4du)__a ^ (__v4du)__b); @@ -687,7 +679,7 @@ _mm256_xor_pd(__m256d __a, __m256d __b) /// A 256-bit vector of [8 x float] containing one of the source operands. /// \returns A 256-bit vector of [8 x float] containing the bitwise XOR of the /// values between both operands. -static __inline __m256 __DEFAULT_FN_ATTRS +static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_xor_ps(__m256 __a, __m256 __b) { return (__m256)((__v8su)__a ^ (__v8su)__b); @@ -2392,7 +2384,7 @@ _mm256_cvtss_f32(__m256 __a) /// return value. /// \returns A 256-bit vector of [8 x float] containing the moved and duplicated /// values. -static __inline __m256 __DEFAULT_FN_ATTRS +static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_movehdup_ps(__m256 __a) { return __builtin_shufflevector((__v8sf)__a, (__v8sf)__a, 1, 1, 3, 3, 5, 5, 7, 7); @@ -2417,7 +2409,7 @@ _mm256_movehdup_ps(__m256 __a) /// return value. /// \returns A 256-bit vector of [8 x float] containing the moved and duplicated /// values. -static __inline __m256 __DEFAULT_FN_ATTRS +static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_moveldup_ps(__m256 __a) { return __builtin_shufflevector((__v8sf)__a, (__v8sf)__a, 0, 0, 2, 2, 4, 4, 6, 6); @@ -2439,7 +2431,7 @@ _mm256_moveldup_ps(__m256 __a) /// the return value. /// \returns A 256-bit vector of [4 x double] containing the moved and /// duplicated values. -static __inline __m256d __DEFAULT_FN_ATTRS +static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_movedup_pd(__m256d __a) { return __builtin_shufflevector((__v4df)__a, (__v4df)__a, 0, 0, 2, 2); @@ -2462,9 +2454,8 @@ _mm256_movedup_pd(__m256d __a) /// Bits [127:64] are written to bits [127:64] of the return value. \n /// Bits [255:192] are written to bits [255:192] of the return value. \n /// \returns A 256-bit vector of [4 x double] containing the interleaved values. -static __inline __m256d __DEFAULT_FN_ATTRS -_mm256_unpackhi_pd(__m256d __a, __m256d __b) -{ +static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR +_mm256_unpackhi_pd(__m256d __a, __m256d __b) { return __builtin_shufflevector((__v4df)__a, (__v4df)__b, 1, 5, 1+2, 5+2); } @@ -2484,9 +2475,8 @@ _mm256_unpackhi_pd(__m256d __a, __m256d __b) /// Bits [63:0] are written to bits [127:64] of the return value. \n /// Bits [191:128] are written to bits [255:192] of the return value. \n /// \returns A 256-bit vector of [4 x double] containing the interleaved values. -static __inline __m256d __DEFAULT_FN_ATTRS -_mm256_unpacklo_pd(__m256d __a, __m256d __b) -{ +static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR +_mm256_unpacklo_pd(__m256d __a, __m256d __b) { return __builtin_shufflevector((__v4df)__a, (__v4df)__b, 0, 4, 0+2, 4+2); } @@ -2511,9 +2501,8 @@ _mm256_unpacklo_pd(__m256d __a, __m256d __b) /// Bits [223:192] are written to bits [191:160] of the return value. \n /// Bits [255:224] are written to bits [255:224] of the return value. /// \returns A 256-bit vector of [8 x float] containing the interleaved values. -static __inline __m256 __DEFAULT_FN_ATTRS -_mm256_unpackhi_ps(__m256 __a, __m256 __b) -{ +static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR +_mm256_unpackhi_ps(__m256 __a, __m256 __b) { return __builtin_shufflevector((__v8sf)__a, (__v8sf)__b, 2, 10, 2+1, 10+1, 6, 14, 6+1, 14+1); } @@ -2538,9 +2527,8 @@ _mm256_unpackhi_ps(__m256 __a, __m256 __b) /// Bits [159:128] are written to bits [191:160] of the return value. \n /// Bits [191:160] are written to bits [255:224] of the return value. /// \returns A 256-bit vector of [8 x float] containing the interleaved values. -static __inline __m256 __DEFAULT_FN_ATTRS -_mm256_unpacklo_ps(__m256 __a, __m256 __b) -{ +static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR +_mm256_unpacklo_ps(__m256 __a, __m256 __b) { return __builtin_shufflevector((__v8sf)__a, (__v8sf)__b, 0, 8, 0+1, 8+1, 4, 12, 4+1, 12+1); } diff --git a/clang/lib/Headers/emmintrin.h b/clang/lib/Headers/emmintrin.h index 78e8a42..770bb5c 100644 --- a/clang/lib/Headers/emmintrin.h +++ b/clang/lib/Headers/emmintrin.h @@ -2127,8 +2127,9 @@ _mm_add_epi32(__m128i __a, __m128i __b) { /// \param __b /// A 64-bit integer. /// \returns A 64-bit integer containing the sum of both parameters. -static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_add_si64(__m64 __a, __m64 __b) { - return (__m64)(((unsigned long long)__a) + ((unsigned long long)__b)); +static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_add_si64(__m64 __a, + __m64 __b) { + return (__m64)(((__v1du)__a)[0] + ((__v1du)__b)[0]); } /// Adds the corresponding elements of two 128-bit vectors of [2 x i64], @@ -2557,8 +2558,9 @@ _mm_sub_epi32(__m128i __a, __m128i __b) { /// A 64-bit integer vector containing the subtrahend. /// \returns A 64-bit integer vector containing the difference of the values in /// the operands. -static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_sub_si64(__m64 __a, __m64 __b) { - return (__m64)((unsigned long long)__a - (unsigned long long)__b); +static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_sub_si64(__m64 __a, + __m64 __b) { + return (__m64)(((__v1du)__a)[0] - ((__v1du)__b)[0]); } /// Subtracts the corresponding elements of two [2 x i64] vectors. @@ -2676,8 +2678,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epu16(__m128i __a, /// A 128-bit integer vector containing one of the source operands. /// \returns A 128-bit integer vector containing the bitwise AND of the values /// in both operands. -static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_and_si128(__m128i __a, - __m128i __b) { +static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm_and_si128(__m128i __a, __m128i __b) { return (__m128i)((__v2du)__a & (__v2du)__b); } @@ -2695,8 +2697,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_and_si128(__m128i __a, /// A 128-bit vector containing the right source operand. /// \returns A 128-bit integer vector containing the bitwise AND of the one's /// complement of the first operand and the values in the second operand. -static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_andnot_si128(__m128i __a, - __m128i __b) { +static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm_andnot_si128(__m128i __a, __m128i __b) { return (__m128i)(~(__v2du)__a & (__v2du)__b); } /// Performs a bitwise OR of two 128-bit integer vectors. @@ -2711,8 +2713,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_andnot_si128(__m128i __a, /// A 128-bit integer vector containing one of the source operands. /// \returns A 128-bit integer vector containing the bitwise OR of the values /// in both operands. -static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_or_si128(__m128i __a, - __m128i __b) { +static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm_or_si128(__m128i __a, __m128i __b) { return (__m128i)((__v2du)__a | (__v2du)__b); } @@ -2728,8 +2730,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_or_si128(__m128i __a, /// A 128-bit integer vector containing one of the source operands. /// \returns A 128-bit integer vector containing the bitwise exclusive OR of the /// values in both operands. -static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_xor_si128(__m128i __a, - __m128i __b) { +static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm_xor_si128(__m128i __a, __m128i __b) { return (__m128i)((__v2du)__a ^ (__v2du)__b); } diff --git a/clang/lib/Headers/mmintrin.h b/clang/lib/Headers/mmintrin.h index dc0fa5c..5a02a455 100644 --- a/clang/lib/Headers/mmintrin.h +++ b/clang/lib/Headers/mmintrin.h @@ -85,7 +85,7 @@ _mm_empty(void) { /// A 32-bit integer value. /// \returns A 64-bit integer vector. The lower 32 bits contain the value of the /// parameter. The upper 32 bits are set to 0. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_cvtsi32_si64(int __i) { return __extension__ (__m64)(__v2si){__i, 0}; @@ -102,7 +102,7 @@ _mm_cvtsi32_si64(int __i) /// A 64-bit integer vector. /// \returns A 32-bit signed integer value containing the lower 32 bits of the /// parameter. -static __inline__ int __DEFAULT_FN_ATTRS_SSE2 +static __inline__ int __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_cvtsi64_si32(__m64 __m) { return ((__v2si)__m)[0]; @@ -118,10 +118,10 @@ _mm_cvtsi64_si32(__m64 __m) /// A 64-bit signed integer. /// \returns A 64-bit integer vector containing the same bitwise pattern as the /// parameter. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_cvtsi64_m64(long long __i) { - return (__m64)__i; + return __extension__ (__m64)(__v1di){__i}; } /// Casts a 64-bit integer vector into a 64-bit signed integer value. @@ -134,10 +134,10 @@ _mm_cvtsi64_m64(long long __i) /// A 64-bit integer vector. /// \returns A 64-bit signed integer containing the same bitwise pattern as the /// parameter. -static __inline__ long long __DEFAULT_FN_ATTRS_SSE2 +static __inline__ long long __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_cvtm64_si64(__m64 __m) { - return (long long)__m; + return ((__v1di)__m)[0]; } /// Converts, with saturation, 16-bit signed integers from both 64-bit integer @@ -379,7 +379,7 @@ _mm_unpacklo_pi32(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [8 x i8]. /// \returns A 64-bit integer vector of [8 x i8] containing the sums of both /// parameters. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_add_pi8(__m64 __m1, __m64 __m2) { return (__m64)(((__v8qu)__m1) + ((__v8qu)__m2)); @@ -400,7 +400,7 @@ _mm_add_pi8(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [4 x i16]. /// \returns A 64-bit integer vector of [4 x i16] containing the sums of both /// parameters. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_add_pi16(__m64 __m1, __m64 __m2) { return (__m64)(((__v4hu)__m1) + ((__v4hu)__m2)); @@ -421,7 +421,7 @@ _mm_add_pi16(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [2 x i32]. /// \returns A 64-bit integer vector of [2 x i32] containing the sums of both /// parameters. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_add_pi32(__m64 __m1, __m64 __m2) { return (__m64)(((__v2su)__m1) + ((__v2su)__m2)); @@ -536,7 +536,7 @@ _mm_adds_pu16(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [8 x i8] containing the subtrahends. /// \returns A 64-bit integer vector of [8 x i8] containing the differences of /// both parameters. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_sub_pi8(__m64 __m1, __m64 __m2) { return (__m64)(((__v8qu)__m1) - ((__v8qu)__m2)); @@ -557,7 +557,7 @@ _mm_sub_pi8(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [4 x i16] containing the subtrahends. /// \returns A 64-bit integer vector of [4 x i16] containing the differences of /// both parameters. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_sub_pi16(__m64 __m1, __m64 __m2) { return (__m64)(((__v4hu)__m1) - ((__v4hu)__m2)); @@ -578,7 +578,7 @@ _mm_sub_pi16(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [2 x i32] containing the subtrahends. /// \returns A 64-bit integer vector of [2 x i32] containing the differences of /// both parameters. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_sub_pi32(__m64 __m1, __m64 __m2) { return (__m64)(((__v2su)__m1) - ((__v2su)__m2)); @@ -745,7 +745,7 @@ _mm_mulhi_pi16(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [4 x i16]. /// \returns A 64-bit integer vector of [4 x i16] containing the lower 16 bits /// of the products of both parameters. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_mullo_pi16(__m64 __m1, __m64 __m2) { return (__m64)(((__v4hu)__m1) * ((__v4hu)__m2)); @@ -1134,7 +1134,7 @@ _mm_srli_si64(__m64 __m, int __count) /// A 64-bit integer vector. /// \returns A 64-bit integer vector containing the bitwise AND of both /// parameters. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_and_si64(__m64 __m1, __m64 __m2) { return (__m64)(((__v1du)__m1) & ((__v1du)__m2)); @@ -1155,7 +1155,7 @@ _mm_and_si64(__m64 __m1, __m64 __m2) /// A 64-bit integer vector. /// \returns A 64-bit integer vector containing the bitwise AND of the second /// parameter and the one's complement of the first parameter. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_andnot_si64(__m64 __m1, __m64 __m2) { return (__m64)(~((__v1du)__m1) & ((__v1du)__m2)); @@ -1173,7 +1173,7 @@ _mm_andnot_si64(__m64 __m1, __m64 __m2) /// A 64-bit integer vector. /// \returns A 64-bit integer vector containing the bitwise OR of both /// parameters. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_or_si64(__m64 __m1, __m64 __m2) { return (__m64)(((__v1du)__m1) | ((__v1du)__m2)); @@ -1191,7 +1191,7 @@ _mm_or_si64(__m64 __m1, __m64 __m2) /// A 64-bit integer vector. /// \returns A 64-bit integer vector containing the bitwise exclusive OR of both /// parameters. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_xor_si64(__m64 __m1, __m64 __m2) { return (__m64)(((__v1du)__m1) ^ ((__v1du)__m2)); @@ -1213,7 +1213,7 @@ _mm_xor_si64(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [8 x i8]. /// \returns A 64-bit integer vector of [8 x i8] containing the comparison /// results. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_cmpeq_pi8(__m64 __m1, __m64 __m2) { return (__m64)(((__v8qi)__m1) == ((__v8qi)__m2)); @@ -1235,7 +1235,7 @@ _mm_cmpeq_pi8(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [4 x i16]. /// \returns A 64-bit integer vector of [4 x i16] containing the comparison /// results. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_cmpeq_pi16(__m64 __m1, __m64 __m2) { return (__m64)(((__v4hi)__m1) == ((__v4hi)__m2)); @@ -1257,7 +1257,7 @@ _mm_cmpeq_pi16(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [2 x i32]. /// \returns A 64-bit integer vector of [2 x i32] containing the comparison /// results. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_cmpeq_pi32(__m64 __m1, __m64 __m2) { return (__m64)(((__v2si)__m1) == ((__v2si)__m2)); @@ -1279,7 +1279,7 @@ _mm_cmpeq_pi32(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [8 x i8]. /// \returns A 64-bit integer vector of [8 x i8] containing the comparison /// results. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_cmpgt_pi8(__m64 __m1, __m64 __m2) { /* This function always performs a signed comparison, but __v8qi is a char @@ -1303,7 +1303,7 @@ _mm_cmpgt_pi8(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [4 x i16]. /// \returns A 64-bit integer vector of [4 x i16] containing the comparison /// results. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_cmpgt_pi16(__m64 __m1, __m64 __m2) { return (__m64)((__v4hi)__m1 > (__v4hi)__m2); @@ -1325,7 +1325,7 @@ _mm_cmpgt_pi16(__m64 __m1, __m64 __m2) /// A 64-bit integer vector of [2 x i32]. /// \returns A 64-bit integer vector of [2 x i32] containing the comparison /// results. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR _mm_cmpgt_pi32(__m64 __m1, __m64 __m2) { return (__m64)((__v2si)__m1 > (__v2si)__m2); diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp index 62bc321..65ff902 100644 --- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp @@ -840,20 +840,27 @@ ProgramStateRef RetainCountChecker::updateSymbol(ProgramStateRef state, const RefCountBug & RetainCountChecker::errorKindToBugKind(RefVal::Kind ErrorKind, SymbolRef Sym) const { + const RefCountFrontend &FE = getPreferredFrontend(); + switch (ErrorKind) { case RefVal::ErrorUseAfterRelease: - return *UseAfterRelease; + return FE.UseAfterRelease; case RefVal::ErrorReleaseNotOwned: - return *ReleaseNotOwned; + return FE.ReleaseNotOwned; case RefVal::ErrorDeallocNotOwned: if (Sym->getType()->getPointeeCXXRecordDecl()) - return *FreeNotOwned; - return *DeallocNotOwned; + return FE.FreeNotOwned; + return FE.DeallocNotOwned; default: llvm_unreachable("Unhandled error."); } } +bool RetainCountChecker::isReleaseUnownedError(RefVal::Kind ErrorKind) const { + return ErrorKind == RefVal::ErrorReleaseNotOwned || + ErrorKind == RefVal::ErrorDeallocNotOwned; +} + void RetainCountChecker::processNonLeakError(ProgramStateRef St, SourceRange ErrorRange, RefVal::Kind ErrorKind, @@ -874,8 +881,8 @@ void RetainCountChecker::processNonLeakError(ProgramStateRef St, return; auto report = std::make_unique<RefCountReport>( - errorKindToBugKind(ErrorKind, Sym), - C.getASTContext().getLangOpts(), N, Sym); + errorKindToBugKind(ErrorKind, Sym), C.getASTContext().getLangOpts(), N, + Sym, /*isLeak=*/false, isReleaseUnownedError(ErrorKind)); report->addRange(ErrorRange); C.emitReport(std::move(report)); } @@ -1090,8 +1097,8 @@ ExplodedNode * RetainCountChecker::checkReturnWithRetEffect(const ReturnStmt *S, ExplodedNode *N = C.addTransition(state, Pred); if (N) { const LangOptions &LOpts = C.getASTContext().getLangOpts(); - auto R = - std::make_unique<RefLeakReport>(*LeakAtReturn, LOpts, N, Sym, C); + auto R = std::make_unique<RefLeakReport>( + getPreferredFrontend().LeakAtReturn, LOpts, N, Sym, C); C.emitReport(std::move(R)); } return N; @@ -1113,7 +1120,8 @@ ExplodedNode * RetainCountChecker::checkReturnWithRetEffect(const ReturnStmt *S, ExplodedNode *N = C.addTransition(state, Pred); if (N) { auto R = std::make_unique<RefCountReport>( - *ReturnNotOwnedForOwned, C.getASTContext().getLangOpts(), N, Sym); + getPreferredFrontend().ReturnNotOwnedForOwned, + C.getASTContext().getLangOpts(), N, Sym); C.emitReport(std::move(R)); } return N; @@ -1261,8 +1269,8 @@ ProgramStateRef RetainCountChecker::handleAutoreleaseCounts( os << "has a +" << V.getCount() << " retain count"; const LangOptions &LOpts = Ctx.getASTContext().getLangOpts(); - auto R = std::make_unique<RefCountReport>(*OverAutorelease, LOpts, N, Sym, - os.str()); + auto R = std::make_unique<RefCountReport>( + getPreferredFrontend().OverAutorelease, LOpts, N, Sym, os.str()); Ctx.emitReport(std::move(R)); } @@ -1307,8 +1315,10 @@ RetainCountChecker::processLeaks(ProgramStateRef state, const LangOptions &LOpts = Ctx.getASTContext().getLangOpts(); if (N) { + const RefCountFrontend &FE = getPreferredFrontend(); + const RefCountBug &BT = Pred ? FE.LeakWithinFunction : FE.LeakAtReturn; + for (SymbolRef L : Leaked) { - const RefCountBug &BT = Pred ? *LeakWithinFunction : *LeakAtReturn; Ctx.emitReport(std::make_unique<RefLeakReport>(BT, LOpts, N, L, Ctx)); } } @@ -1463,44 +1473,31 @@ std::unique_ptr<SimpleProgramPointTag> RetainCountChecker::DeallocSentTag; std::unique_ptr<SimpleProgramPointTag> RetainCountChecker::CastFailTag; void ento::registerRetainCountBase(CheckerManager &Mgr) { - auto *Chk = Mgr.registerChecker<RetainCountChecker>(); + auto *Chk = Mgr.getChecker<RetainCountChecker>(); Chk->DeallocSentTag = std::make_unique<SimpleProgramPointTag>( "RetainCountChecker", "DeallocSent"); Chk->CastFailTag = std::make_unique<SimpleProgramPointTag>( "RetainCountChecker", "DynamicCastFail"); } -bool ento::shouldRegisterRetainCountBase(const CheckerManager &mgr) { +bool ento::shouldRegisterRetainCountBase(const CheckerManager &) { return true; } + void ento::registerRetainCountChecker(CheckerManager &Mgr) { auto *Chk = Mgr.getChecker<RetainCountChecker>(); - Chk->TrackObjCAndCFObjects = true; + Chk->RetainCount.enable(Mgr); Chk->TrackNSCFStartParam = Mgr.getAnalyzerOptions().getCheckerBooleanOption( Mgr.getCurrentCheckerName(), "TrackNSCFStartParam"); - -#define INIT_BUGTYPE(KIND) \ - Chk->KIND = std::make_unique<RefCountBug>(Mgr.getCurrentCheckerName(), \ - RefCountBug::KIND); - // TODO: Ideally, we should have a checker for each of these bug types. - INIT_BUGTYPE(UseAfterRelease) - INIT_BUGTYPE(ReleaseNotOwned) - INIT_BUGTYPE(DeallocNotOwned) - INIT_BUGTYPE(FreeNotOwned) - INIT_BUGTYPE(OverAutorelease) - INIT_BUGTYPE(ReturnNotOwnedForOwned) - INIT_BUGTYPE(LeakWithinFunction) - INIT_BUGTYPE(LeakAtReturn) -#undef INIT_BUGTYPE } -bool ento::shouldRegisterRetainCountChecker(const CheckerManager &mgr) { +bool ento::shouldRegisterRetainCountChecker(const CheckerManager &) { return true; } void ento::registerOSObjectRetainCountChecker(CheckerManager &Mgr) { auto *Chk = Mgr.getChecker<RetainCountChecker>(); - Chk->TrackOSObjects = true; + Chk->OSObjectRetainCount.enable(Mgr); // FIXME: We want bug reports to always have the same checker name associated // with them, yet here, if RetainCountChecker is disabled but @@ -1511,21 +1508,8 @@ void ento::registerOSObjectRetainCountChecker(CheckerManager &Mgr) { // diagnostics, and **hidden checker options** with the fine-tuning of // modeling. Following this logic, OSObjectRetainCountChecker should be the // latter, but we can't just remove it for backward compatibility reasons. -#define LAZY_INIT_BUGTYPE(KIND) \ - if (!Chk->KIND) \ - Chk->KIND = std::make_unique<RefCountBug>(Mgr.getCurrentCheckerName(), \ - RefCountBug::KIND); - LAZY_INIT_BUGTYPE(UseAfterRelease) - LAZY_INIT_BUGTYPE(ReleaseNotOwned) - LAZY_INIT_BUGTYPE(DeallocNotOwned) - LAZY_INIT_BUGTYPE(FreeNotOwned) - LAZY_INIT_BUGTYPE(OverAutorelease) - LAZY_INIT_BUGTYPE(ReturnNotOwnedForOwned) - LAZY_INIT_BUGTYPE(LeakWithinFunction) - LAZY_INIT_BUGTYPE(LeakAtReturn) -#undef LAZY_INIT_BUGTYPE } -bool ento::shouldRegisterOSObjectRetainCountChecker(const CheckerManager &mgr) { +bool ento::shouldRegisterOSObjectRetainCountChecker(const CheckerManager &) { return true; } diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.h b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.h index 0e81143..8854e10 100644 --- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.h +++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.h @@ -235,51 +235,32 @@ public: }; class RetainCountChecker - : public Checker< check::Bind, - check::DeadSymbols, - check::BeginFunction, - check::EndFunction, - check::PostStmt<BlockExpr>, - check::PostStmt<CastExpr>, - check::PostStmt<ObjCArrayLiteral>, - check::PostStmt<ObjCDictionaryLiteral>, - check::PostStmt<ObjCBoxedExpr>, - check::PostStmt<ObjCIvarRefExpr>, - check::PostCall, - check::RegionChanges, - eval::Assume, - eval::Call > { + : public CheckerFamily< + check::Bind, check::DeadSymbols, check::BeginFunction, + check::EndFunction, check::PostStmt<BlockExpr>, + check::PostStmt<CastExpr>, check::PostStmt<ObjCArrayLiteral>, + check::PostStmt<ObjCDictionaryLiteral>, + check::PostStmt<ObjCBoxedExpr>, check::PostStmt<ObjCIvarRefExpr>, + check::PostCall, check::RegionChanges, eval::Assume, eval::Call> { public: - std::unique_ptr<RefCountBug> UseAfterRelease; - std::unique_ptr<RefCountBug> ReleaseNotOwned; - std::unique_ptr<RefCountBug> DeallocNotOwned; - std::unique_ptr<RefCountBug> FreeNotOwned; - std::unique_ptr<RefCountBug> OverAutorelease; - std::unique_ptr<RefCountBug> ReturnNotOwnedForOwned; - std::unique_ptr<RefCountBug> LeakWithinFunction; - std::unique_ptr<RefCountBug> LeakAtReturn; + RefCountFrontend RetainCount; + RefCountFrontend OSObjectRetainCount; mutable std::unique_ptr<RetainSummaryManager> Summaries; static std::unique_ptr<SimpleProgramPointTag> DeallocSentTag; static std::unique_ptr<SimpleProgramPointTag> CastFailTag; - /// Track Objective-C and CoreFoundation objects. - bool TrackObjCAndCFObjects = false; - - /// Track sublcasses of OSObject. - bool TrackOSObjects = false; - /// Track initial parameters (for the entry point) for NS/CF objects. bool TrackNSCFStartParam = false; - RetainCountChecker() {}; + StringRef getDebugTag() const override { return "RetainCountChecker"; } RetainSummaryManager &getSummaryManager(ASTContext &Ctx) const { if (!Summaries) - Summaries.reset( - new RetainSummaryManager(Ctx, TrackObjCAndCFObjects, TrackOSObjects)); + Summaries = std::make_unique<RetainSummaryManager>( + Ctx, RetainCount.isEnabled(), OSObjectRetainCount.isEnabled()); return *Summaries; } @@ -287,6 +268,15 @@ public: return getSummaryManager(C.getASTContext()); } + const RefCountFrontend &getPreferredFrontend() const { + // FIXME: The two frontends of this checker family are in an unusual + // relationship: if they are both enabled, then all bug reports are + // reported by RetainCount (i.e. `osx.cocoa.RetainCount`), even the bugs + // that "belong to" OSObjectRetainCount (i.e. `osx.OSObjectRetainCount`). + // This is counter-intuitive and should be fixed to avoid confusion. + return RetainCount.isEnabled() ? RetainCount : OSObjectRetainCount; + } + void printState(raw_ostream &Out, ProgramStateRef State, const char *NL, const char *Sep) const override; @@ -337,6 +327,8 @@ public: const RefCountBug &errorKindToBugKind(RefVal::Kind ErrorKind, SymbolRef Sym) const; + bool isReleaseUnownedError(RefVal::Kind ErrorKind) const; + void processNonLeakError(ProgramStateRef St, SourceRange ErrorRange, RefVal::Kind ErrorKind, SymbolRef Sym, CheckerContext &C) const; diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp index c9f5dc9..cad2c72 100644 --- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp @@ -21,57 +21,6 @@ using namespace clang; using namespace ento; using namespace retaincountchecker; -StringRef RefCountBug::bugTypeToName(RefCountBug::RefCountBugKind BT) { - switch (BT) { - case UseAfterRelease: - return "Use-after-release"; - case ReleaseNotOwned: - return "Bad release"; - case DeallocNotOwned: - return "-dealloc sent to non-exclusively owned object"; - case FreeNotOwned: - return "freeing non-exclusively owned object"; - case OverAutorelease: - return "Object autoreleased too many times"; - case ReturnNotOwnedForOwned: - return "Method should return an owned object"; - case LeakWithinFunction: - return "Leak"; - case LeakAtReturn: - return "Leak of returned object"; - } - llvm_unreachable("Unknown RefCountBugKind"); -} - -StringRef RefCountBug::getDescription() const { - switch (BT) { - case UseAfterRelease: - return "Reference-counted object is used after it is released"; - case ReleaseNotOwned: - return "Incorrect decrement of the reference count of an object that is " - "not owned at this point by the caller"; - case DeallocNotOwned: - return "-dealloc sent to object that may be referenced elsewhere"; - case FreeNotOwned: - return "'free' called on an object that may be referenced elsewhere"; - case OverAutorelease: - return "Object autoreleased too many times"; - case ReturnNotOwnedForOwned: - return "Object with a +0 retain count returned to caller where a +1 " - "(owning) retain count is expected"; - case LeakWithinFunction: - case LeakAtReturn: - return ""; - } - llvm_unreachable("Unknown RefCountBugKind"); -} - -RefCountBug::RefCountBug(CheckerNameRef Checker, RefCountBugKind BT) - : BugType(Checker, bugTypeToName(BT), categories::MemoryRefCount, - /*SuppressOnSink=*/BT == LeakWithinFunction || - BT == LeakAtReturn), - BT(BT) {} - static bool isNumericLiteralExpression(const Expr *E) { // FIXME: This set of cases was copied from SemaExprObjC. return isa<IntegerLiteral, CharacterLiteral, FloatingLiteral, @@ -312,9 +261,11 @@ namespace retaincountchecker { class RefCountReportVisitor : public BugReporterVisitor { protected: SymbolRef Sym; + bool IsReleaseUnowned; public: - RefCountReportVisitor(SymbolRef sym) : Sym(sym) {} + RefCountReportVisitor(SymbolRef S, bool IRU) + : Sym(S), IsReleaseUnowned(IRU) {} void Profile(llvm::FoldingSetNodeID &ID) const override { static int x = 0; @@ -334,7 +285,8 @@ public: class RefLeakReportVisitor : public RefCountReportVisitor { public: RefLeakReportVisitor(SymbolRef Sym, const MemRegion *LastBinding) - : RefCountReportVisitor(Sym), LastBinding(LastBinding) {} + : RefCountReportVisitor(Sym, /*IsReleaseUnowned=*/false), + LastBinding(LastBinding) {} PathDiagnosticPieceRef getEndPath(BugReporterContext &BRC, const ExplodedNode *N, @@ -452,12 +404,6 @@ annotateStartParameter(const ExplodedNode *N, SymbolRef Sym, PathDiagnosticPieceRef RefCountReportVisitor::VisitNode(const ExplodedNode *N, BugReporterContext &BRC, PathSensitiveBugReport &BR) { - - const auto &BT = static_cast<const RefCountBug&>(BR.getBugType()); - - bool IsFreeUnowned = BT.getBugType() == RefCountBug::FreeNotOwned || - BT.getBugType() == RefCountBug::DeallocNotOwned; - const SourceManager &SM = BRC.getSourceManager(); CallEventManager &CEMgr = BRC.getStateManager().getCallEventManager(); if (auto CE = N->getLocationAs<CallExitBegin>()) @@ -490,7 +436,7 @@ RefCountReportVisitor::VisitNode(const ExplodedNode *N, BugReporterContext &BRC, std::string sbuf; llvm::raw_string_ostream os(sbuf); - if (PrevT && IsFreeUnowned && CurrV.isNotOwned() && PrevT->isOwned()) { + if (PrevT && IsReleaseUnowned && CurrV.isNotOwned() && PrevT->isOwned()) { os << "Object is now not exclusively owned"; auto Pos = PathDiagnosticLocation::create(N->getLocation(), SM); return std::make_shared<PathDiagnosticEventPiece>(Pos, sbuf); @@ -815,10 +761,8 @@ RefLeakReportVisitor::getEndPath(BugReporterContext &BRC, if (K == ObjKind::ObjC || K == ObjKind::CF) { os << "whose name ('" << *FD << "') does not contain 'Copy' or 'Create'. This violates the " - "naming" - " convention rules given in the Memory Management Guide for " - "Core" - " Foundation"; + "naming convention rules given in the Memory Management Guide " + "for Core Foundation"; } else if (RV->getObjKind() == ObjKind::OS) { std::string FuncName = FD->getNameAsString(); os << "whose name ('" << FuncName << "') starts with '" @@ -836,19 +780,20 @@ RefLeakReportVisitor::getEndPath(BugReporterContext &BRC, } RefCountReport::RefCountReport(const RefCountBug &D, const LangOptions &LOpts, - ExplodedNode *n, SymbolRef sym, bool isLeak) - : PathSensitiveBugReport(D, D.getDescription(), n), Sym(sym), + ExplodedNode *n, SymbolRef sym, bool isLeak, + bool IsReleaseUnowned) + : PathSensitiveBugReport(D, D.getReportMessage(), n), Sym(sym), isLeak(isLeak) { if (!isLeak) - addVisitor<RefCountReportVisitor>(sym); + addVisitor<RefCountReportVisitor>(sym, IsReleaseUnowned); } RefCountReport::RefCountReport(const RefCountBug &D, const LangOptions &LOpts, ExplodedNode *n, SymbolRef sym, StringRef endText) - : PathSensitiveBugReport(D, D.getDescription(), endText, n) { + : PathSensitiveBugReport(D, D.getReportMessage(), endText, n) { - addVisitor<RefCountReportVisitor>(sym); + addVisitor<RefCountReportVisitor>(sym, /*IsReleaseUnowned=*/false); } void RefLeakReport::deriveParamLocation(CheckerContext &Ctx) { diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.h b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.h index d059008..6ceb86f 100644 --- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.h +++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.h @@ -25,25 +25,44 @@ namespace ento { namespace retaincountchecker { class RefCountBug : public BugType { + StringRef ReportMessage; + public: - enum RefCountBugKind { - UseAfterRelease, - ReleaseNotOwned, - DeallocNotOwned, - FreeNotOwned, - OverAutorelease, - ReturnNotOwnedForOwned, - LeakWithinFunction, - LeakAtReturn, - }; - RefCountBug(CheckerNameRef Checker, RefCountBugKind BT); - StringRef getDescription() const; - - RefCountBugKind getBugType() const { return BT; } - -private: - RefCountBugKind BT; - static StringRef bugTypeToName(RefCountBugKind BT); + RefCountBug(const CheckerFrontend *CF, StringRef Desc, StringRef ReportMsg, + bool SuppressOnSink = false) + : BugType(CF, Desc, categories::MemoryRefCount, SuppressOnSink), + ReportMessage(ReportMsg) {} + StringRef getReportMessage() const { return ReportMessage; } +}; + +class RefCountFrontend : public CheckerFrontend { +public: + const RefCountBug UseAfterRelease{ + this, "Use-after-release", + "Reference-counted object is used after it is released"}; + const RefCountBug ReleaseNotOwned{ + this, "Bad release", + "Incorrect decrement of the reference count of an object that is not " + "owned at this point by the caller"}; + const RefCountBug DeallocNotOwned{ + this, "-dealloc sent to non-exclusively owned object", + "-dealloc sent to object that may be referenced elsewhere"}; + const RefCountBug FreeNotOwned{ + this, "freeing non-exclusively owned object", + "'free' called on an object that may be referenced elsewhere"}; + const RefCountBug OverAutorelease{this, "Object autoreleased too many times", + "Object autoreleased too many times"}; + const RefCountBug ReturnNotOwnedForOwned{ + this, "Method should return an owned object", + "Object with a +0 retain count returned to caller where a +1 (owning) " + "retain count is expected"}; + // For these two bug types the report message will be generated dynamically + // by `RefLeakReport::createDescription` so the empty string taken from the + // BugType will be ignored (overwritten). + const RefCountBug LeakWithinFunction{this, "Leak", /*ReportMsg=*/"", + /*SuppressOnSink=*/true}; + const RefCountBug LeakAtReturn{this, "Leak of returned object", + /*ReportMsg=*/"", /*SuppressOnSink=*/true}; }; class RefCountReport : public PathSensitiveBugReport { @@ -53,8 +72,8 @@ protected: public: RefCountReport(const RefCountBug &D, const LangOptions &LOpts, - ExplodedNode *n, SymbolRef sym, - bool isLeak=false); + ExplodedNode *n, SymbolRef sym, bool isLeak = false, + bool IsReleaseUnowned = false); RefCountReport(const RefCountBug &D, const LangOptions &LOpts, ExplodedNode *n, SymbolRef sym, diff --git a/clang/test/CodeGen/X86/avx-builtins.c b/clang/test/CodeGen/X86/avx-builtins.c index a6e70aae..e2c9f96 100644 --- a/clang/test/CodeGen/X86/avx-builtins.c +++ b/clang/test/CodeGen/X86/avx-builtins.c @@ -20,12 +20,14 @@ __m256d test_mm256_add_pd(__m256d A, __m256d B) { // CHECK: fadd <4 x double> return _mm256_add_pd(A, B); } +TEST_CONSTEXPR(match_m256d( _mm256_add_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){-4.0, -5.0, +6.0, +7.0}), -8.0, -10.0, +12.0, +14.0)); __m256 test_mm256_add_ps(__m256 A, __m256 B) { // CHECK-LABEL: test_mm256_add_ps // CHECK: fadd <8 x float> return _mm256_add_ps(A, B); } +TEST_CONSTEXPR(match_m256(_mm256_add_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}), -8.0f, -10.0f, +12.0f, +14.0f, +14.0f, +12.0f, -10.0f, -8.0f)); __m256d test_mm256_addsub_pd(__m256d A, __m256d B) { // CHECK-LABEL: test_mm256_addsub_pd @@ -44,12 +46,14 @@ __m256d test_mm256_and_pd(__m256d A, __m256d B) { // CHECK: and <4 x i64> return _mm256_and_pd(A, B); } +TEST_CONSTEXPR(match_m256d(_mm256_and_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){+0.0, -0.0, -0.0, +7.0}), -0.0, -0.0, +0.0, +7.0)); __m256 test_mm256_and_ps(__m256 A, __m256 B) { // CHECK-LABEL: test_mm256_and_ps // CHECK: and <8 x i32> return _mm256_and_ps(A, B); } +TEST_CONSTEXPR(match_m256(_mm256_and_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), -0.0f, -0.0f, +0.0f, +7.0f, +7.0f, +0.0f, -0.0f, -0.0f)); __m256d test_mm256_andnot_pd(__m256d A, __m256d B) { // CHECK-LABEL: test_mm256_andnot_pd @@ -57,6 +61,7 @@ __m256d test_mm256_andnot_pd(__m256d A, __m256d B) { // CHECK: and <4 x i64> return _mm256_andnot_pd(A, B); } +TEST_CONSTEXPR(match_m256d(_mm256_andnot_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){+0.0, -0.0, -0.0, +7.0}), +0.0, +0.0, +0.0, +0.0)); __m256 test_mm256_andnot_ps(__m256 A, __m256 B) { // CHECK-LABEL: test_mm256_andnot_ps @@ -64,6 +69,7 @@ __m256 test_mm256_andnot_ps(__m256 A, __m256 B) { // CHECK: and <8 x i32> return _mm256_andnot_ps(A, B); } +TEST_CONSTEXPR(match_m256(_mm256_andnot_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f)); __m256d test_mm256_blend_pd(__m256d A, __m256d B) { // CHECK-LABEL: test_mm256_blend_pd @@ -973,12 +979,14 @@ __m256d test_mm256_div_pd(__m256d A, __m256d B) { // CHECK: fdiv <4 x double> return _mm256_div_pd(A, B); } +TEST_CONSTEXPR(match_m256d( _mm256_div_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){-1.0, +1.0, -1.0, +1.0}), +4.0, -5.0, -6.0, +7.0)); __m256 test_mm256_div_ps(__m256 A, __m256 B) { // CHECK-LABEL: test_mm256_div_ps // CHECK: fdiv <8 x float> return _mm256_div_ps(A, B); } +TEST_CONSTEXPR(match_m256( _mm256_div_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){-1.0f, +1.0f, -1.0f, +1.0f, +1.0f, -1.0f, +1.0f, -1.0f}), +4.0f, -5.0f, -6.0f, +7.0f, +7.0f, -6.0f, -5.0f, +4.0f)); __m256 test_mm256_dp_ps(__m256 A, __m256 B) { // CHECK-LABEL: test_mm256_dp_ps @@ -1258,18 +1266,21 @@ __m256d test_mm256_movedup_pd(__m256d A) { // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 0, i32 2, i32 2> return _mm256_movedup_pd(A); } +TEST_CONSTEXPR(match_m256d(_mm256_movedup_pd((__m256d){+7.0, -7.0, -42.0, +42.0}), +7.0, +7.0, -42.0, -42.0)); __m256 test_mm256_movehdup_ps(__m256 A) { // CHECK-LABEL: test_mm256_movehdup_ps // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7> return _mm256_movehdup_ps(A); } +TEST_CONSTEXPR(match_m256(_mm256_movehdup_ps((__m256){+1.0f,-1.0f,+2.0f,+4.0f,+8.0f,-8.0f,-3.0f,+3.0f}), -1.0f, -1.0f, +4.0f, +4.0f, -8.0f, -8.0f, +3.0f, +3.0f)); __m256 test_mm256_moveldup_ps(__m256 A) { // CHECK-LABEL: test_mm256_moveldup_ps // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6> return _mm256_moveldup_ps(A); } +TEST_CONSTEXPR(match_m256(_mm256_moveldup_ps((__m256){+1.0f,-1.0f,+2.0f,+4.0f,+8.0f,-8.0f,-3.0f,+3.0f}), +1.0f, +1.0f, +2.0f, +2.0f, +8.0f, +8.0f, -3.0f, -3.0f)); int test_mm256_movemask_pd(__m256d A) { // CHECK-LABEL: test_mm256_movemask_pd @@ -1288,24 +1299,28 @@ __m256d test_mm256_mul_pd(__m256d A, __m256d B) { // CHECK: fmul <4 x double> return _mm256_mul_pd(A, B); } +TEST_CONSTEXPR(match_m256d( _mm256_mul_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){-4.0, -5.0, +6.0, +7.0}), +16.0, +25.0, +36.0, +49.0)); __m256 test_mm256_mul_ps(__m256 A, __m256 B) { // CHECK-LABEL: test_mm256_mul_ps // CHECK: fmul <8 x float> return _mm256_mul_ps(A, B); } +TEST_CONSTEXPR(match_m256( _mm256_mul_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}), +16.0f, +25.0f, +36.0f, +49.0f, +49.0f, +36.0f, +25.0f, +16.0f)); __m256d test_mm256_or_pd(__m256d A, __m256d B) { // CHECK-LABEL: test_mm256_or_pd // CHECK: or <4 x i64> return _mm256_or_pd(A, B); } +TEST_CONSTEXPR(match_m256d(_mm256_or_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){+0.0, -0.0, -0.0, +7.0}), -4.0, -5.0, -6.0, +7.0)); __m256 test_mm256_or_ps(__m256 A, __m256 B) { // CHECK-LABEL: test_mm256_or_ps // CHECK: or <8 x i32> return _mm256_or_ps(A, B); } +TEST_CONSTEXPR(match_m256(_mm256_or_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), -4.0f, -5.0f, -6.0f, +7.0f, +7.0f, -6.0f, -5.0f, -4.0f)); __m128d test_mm_permute_pd(__m128d A) { // CHECK-LABEL: test_mm_permute_pd @@ -1924,12 +1939,14 @@ __m256d test_mm256_sub_pd(__m256d A, __m256d B) { // CHECK: fsub <4 x double> return _mm256_sub_pd(A, B); } +TEST_CONSTEXPR(match_m256d( _mm256_sub_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){-0.0, +0.0, +2.0, -1.0}), -4.0, -5.0, 4.0, 8.0)); __m256 test_mm256_sub_ps(__m256 A, __m256 B) { // CHECK-LABEL: test_mm256_sub_ps // CHECK: fsub <8 x float> return _mm256_sub_ps(A, B); } +TEST_CONSTEXPR(match_m256( _mm256_sub_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){-0.0f, +0.0f, +2.0f, -1.0f, -1.0f, +2.0f, +0.0f, -0.0f}), -4.0f, -5.0f, 4.0f, 8.0f, 8.0f, 4.0f, -5.0f, -4.0f)); int test_mm_testc_pd(__m128d A, __m128d B) { // CHECK-LABEL: test_mm_testc_pd @@ -2053,36 +2070,42 @@ __m256d test_mm256_unpackhi_pd(__m256d A, __m256d B) { // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 1, i32 5, i32 3, i32 7> return _mm256_unpackhi_pd(A, B); } +TEST_CONSTEXPR(match_m256d(_mm256_unpackhi_pd((__m256d){+1.0, +2.0, +3.0, +4.0}, (__m256d){+5.0, +6.0, +7.0, +8.0}), +2.0, +6.0, +4.0, +8.0)); __m256 test_mm256_unpackhi_ps(__m256 A, __m256 B) { // CHECK-LABEL: test_mm256_unpackhi_ps // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15> return _mm256_unpackhi_ps(A, B); } +TEST_CONSTEXPR(match_m256(_mm256_unpackhi_ps((__m256){+0.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f}, (__m256){+10.0f, +11.0f, +12.0f, +13.0f, +14.0f, +15.0f, +16.0f, +17.0f}), +2.0f, +12.0f, +3.0f, +13.0f, +6.0f, +16.0f, +7.0f, +17.0f)); __m256d test_mm256_unpacklo_pd(__m256d A, __m256d B) { // CHECK-LABEL: test_mm256_unpacklo_pd // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 2, i32 6> return _mm256_unpacklo_pd(A, B); } +TEST_CONSTEXPR(match_m256d(_mm256_unpacklo_pd((__m256d){+1.0, +2.0, +3.0, +4.0}, (__m256d){+5.0, +6.0, +7.0, +8.0}), +1.0, +5.0, +3.0, +7.0)); __m256 test_mm256_unpacklo_ps(__m256 A, __m256 B) { // CHECK-LABEL: test_mm256_unpacklo_ps // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13> return _mm256_unpacklo_ps(A, B); } +TEST_CONSTEXPR(match_m256(_mm256_unpacklo_ps((__m256){+0.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f}, (__m256){+10.0f, +11.0f, +12.0f, +13.0f, +14.0f, +15.0f, +16.0f, +17.0f}), +0.0f, +10.0f, +1.0f, +11.0f, +4.0f, +14.0f, +5.0f, +15.0f)); __m256d test_mm256_xor_pd(__m256d A, __m256d B) { // CHECK-LABEL: test_mm256_xor_pd // CHECK: xor <4 x i64> return _mm256_xor_pd(A, B); } +TEST_CONSTEXPR(match_m256d(_mm256_xor_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){+0.0, -0.0, -0.0, +7.0}), -4.0, +5.0, -6.0, +0.0)); __m256 test_mm256_xor_ps(__m256 A, __m256 B) { // CHECK-LABEL: test_mm256_xor_ps // CHECK: xor <8 x i32> return _mm256_xor_ps(A, B); } +TEST_CONSTEXPR(match_m256(_mm256_xor_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), -4.0f, +5.0f, -6.0f, +0.0f, +0.0f, -6.0f, +5.0f, -4.0f)); void test_mm256_zeroall(void) { // CHECK-LABEL: test_mm256_zeroall diff --git a/clang/test/CodeGen/X86/avx2-builtins.c b/clang/test/CodeGen/X86/avx2-builtins.c index 27da56f..1ed624c 100644 --- a/clang/test/CodeGen/X86/avx2-builtins.c +++ b/clang/test/CodeGen/X86/avx2-builtins.c @@ -9,6 +9,7 @@ #include <immintrin.h> +#include "builtin_test_helpers.h" // NOTE: This should match the tests in llvm/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll @@ -97,6 +98,7 @@ __m256i test_mm256_and_si256(__m256i a, __m256i b) { // CHECK: and <4 x i64> return _mm256_and_si256(a, b); } +TEST_CONSTEXPR(match_v4di(_mm256_and_si256((__m256i)(__v4di){0, -1, 0, -1}, (__m256i)(__v4di){0, 0, -1, -1}), 0, 0, 0, -1)); __m256i test_mm256_andnot_si256(__m256i a, __m256i b) { // CHECK-LABEL: test_mm256_andnot_si256 @@ -104,6 +106,7 @@ __m256i test_mm256_andnot_si256(__m256i a, __m256i b) { // CHECK: and <4 x i64> return _mm256_andnot_si256(a, b); } +TEST_CONSTEXPR(match_v4di(_mm256_andnot_si256((__m256i)(__v4di){0, -1, 0, -1}, (__m256i)(__v4di){0, 0, -1, -1}), 0, 0, -1, 0)); __m256i test_mm256_avg_epu8(__m256i a, __m256i b) { // CHECK-LABEL: test_mm256_avg_epu8 @@ -920,6 +923,7 @@ __m256i test_mm256_or_si256(__m256i a, __m256i b) { // CHECK: or <4 x i64> return _mm256_or_si256(a, b); } +TEST_CONSTEXPR(match_v4di(_mm256_or_si256((__m256i)(__v4di){0, -1, 0, -1}, (__m256i)(__v4di){0, 0, -1, -1}), 0, -1, -1, -1)); __m256i test_mm256_packs_epi16(__m256i a, __m256i b) { // CHECK-LABEL: test_mm256_packs_epi16 @@ -1336,3 +1340,4 @@ __m256i test_mm256_xor_si256(__m256i a, __m256i b) { // CHECK: xor <4 x i64> return _mm256_xor_si256(a, b); } +TEST_CONSTEXPR(match_v4di(_mm256_xor_si256((__m256i)(__v4di){0, -1, 0, -1}, (__m256i)(__v4di){0, 0, -1, -1}), 0, -1, -1, 0)); diff --git a/clang/test/CodeGen/X86/avx512-reduceIntrin.c b/clang/test/CodeGen/X86/avx512-reduceIntrin.c index 2ceac3a..0f781fc 100644 --- a/clang/test/CodeGen/X86/avx512-reduceIntrin.c +++ b/clang/test/CodeGen/X86/avx512-reduceIntrin.c @@ -1,30 +1,35 @@ // RUN: %clang_cc1 -ffreestanding %s -O0 -triple=x86_64-apple-darwin -target-cpu skylake-avx512 -emit-llvm -o - -Wall -Werror | FileCheck %s #include <immintrin.h> +#include "builtin_test_helpers.h" long long test_mm512_reduce_add_epi64(__m512i __W){ // CHECK-LABEL: @test_mm512_reduce_add_epi64( // CHECK: call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %{{.*}}) return _mm512_reduce_add_epi64(__W); } +TEST_CONSTEXPR(_mm512_reduce_add_epi64((__m512i)(__v8di){-4, -3, -2, -1, 0, 1, 2, 3}) == -4); long long test_mm512_reduce_mul_epi64(__m512i __W){ // CHECK-LABEL: @test_mm512_reduce_mul_epi64( // CHECK: call i64 @llvm.vector.reduce.mul.v8i64(<8 x i64> %{{.*}}) return _mm512_reduce_mul_epi64(__W); } +TEST_CONSTEXPR(_mm512_reduce_mul_epi64((__m512i)(__v8di){1, 2, 3, 4, 5, 6, 7, 8}) == 40320); long long test_mm512_reduce_or_epi64(__m512i __W){ // CHECK-LABEL: @test_mm512_reduce_or_epi64( // CHECK: call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> %{{.*}}) return _mm512_reduce_or_epi64(__W); } +TEST_CONSTEXPR(_mm512_reduce_or_epi64((__m512i)(__v8di){0x100, 0x200, 0x400, 0x800, 0, 0, 0, 0}) == 0xF00); long long test_mm512_reduce_and_epi64(__m512i __W){ // CHECK-LABEL: @test_mm512_reduce_and_epi64( // CHECK: call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> %{{.*}}) return _mm512_reduce_and_epi64(__W); } +TEST_CONSTEXPR(_mm512_reduce_and_epi64((__m512i)(__v8di){0xFFFF, 0xFF00, 0x00FF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFF00, 0x00FF}) == 0x0000); long long test_mm512_mask_reduce_add_epi64(__mmask8 __M, __m512i __W){ // CHECK-LABEL: @test_mm512_mask_reduce_add_epi64( @@ -59,23 +64,27 @@ int test_mm512_reduce_add_epi32(__m512i __W){ // CHECK: call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %{{.*}}) return _mm512_reduce_add_epi32(__W); } +TEST_CONSTEXPR(_mm512_reduce_add_epi32((__m512i)(__v16si){-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7}) == -8); int test_mm512_reduce_mul_epi32(__m512i __W){ // CHECK-LABEL: @test_mm512_reduce_mul_epi32( // CHECK: call i32 @llvm.vector.reduce.mul.v16i32(<16 x i32> %{{.*}}) return _mm512_reduce_mul_epi32(__W); } +TEST_CONSTEXPR(_mm512_reduce_mul_epi32((__m512i)(__v16si){1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 1, 1, -3, 1, 1}) == -36); int test_mm512_reduce_or_epi32(__m512i __W){ // CHECK: call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> %{{.*}}) return _mm512_reduce_or_epi32(__W); } +TEST_CONSTEXPR(_mm512_reduce_or_epi32((__m512i)(__v16si){0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0, 0, 0, 0, 0, 0, 0, 0}) == 0xFF); int test_mm512_reduce_and_epi32(__m512i __W){ // CHECK-LABEL: @test_mm512_reduce_and_epi32( // CHECK: call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> %{{.*}}) return _mm512_reduce_and_epi32(__W); } +TEST_CONSTEXPR(_mm512_reduce_and_epi32((__m512i)(__v16si){0xFF, 0xF0, 0x0F, 0xFF, 0xFF, 0xFF, 0xF0, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xF0, 0xF0, 0x0F, 0x0F}) == 0x00); int test_mm512_mask_reduce_add_epi32(__mmask16 __M, __m512i __W){ // CHECK-LABEL: @test_mm512_mask_reduce_add_epi32( diff --git a/clang/test/CodeGen/X86/avx512-reduceMinMaxIntrin.c b/clang/test/CodeGen/X86/avx512-reduceMinMaxIntrin.c index 3e33ec5..4f7b7b0 100644 --- a/clang/test/CodeGen/X86/avx512-reduceMinMaxIntrin.c +++ b/clang/test/CodeGen/X86/avx512-reduceMinMaxIntrin.c @@ -1,18 +1,21 @@ // RUN: %clang_cc1 -ffreestanding %s -O0 -triple=x86_64-apple-darwin -target-cpu skylake-avx512 -emit-llvm -o - -Wall -Werror | FileCheck %s #include <immintrin.h> +#include "builtin_test_helpers.h" long long test_mm512_reduce_max_epi64(__m512i __W){ // CHECK-LABEL: @test_mm512_reduce_max_epi64( // CHECK: call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> %{{.*}}) return _mm512_reduce_max_epi64(__W); } +TEST_CONSTEXPR(_mm512_reduce_max_epi64((__m512i)(__v8di){-4, -3, -2, -1, 0, 1, 2, 3}) == 3); unsigned long long test_mm512_reduce_max_epu64(__m512i __W){ // CHECK-LABEL: @test_mm512_reduce_max_epu64( // CHECK: call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> %{{.*}}) return _mm512_reduce_max_epu64(__W); } +TEST_CONSTEXPR(_mm512_reduce_max_epu64((__m512i)(__v8du){0, 1, 2, 3, 4, 5, 6, 7}) == 7); double test_mm512_reduce_max_pd(__m512d __W, double ExtraAddOp){ // CHECK-LABEL: @test_mm512_reduce_max_pd( @@ -27,12 +30,14 @@ long long test_mm512_reduce_min_epi64(__m512i __W){ // CHECK: call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> %{{.*}}) return _mm512_reduce_min_epi64(__W); } +TEST_CONSTEXPR(_mm512_reduce_min_epi64((__m512i)(__v8di){-4, -3, -2, -1, 0, 1, 2, 3}) == -4); unsigned long long test_mm512_reduce_min_epu64(__m512i __W){ // CHECK-LABEL: @test_mm512_reduce_min_epu64( // CHECK: call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> %{{.*}}) return _mm512_reduce_min_epu64(__W); } +TEST_CONSTEXPR(_mm512_reduce_min_epu64((__m512i)(__v8du){0, 1, 2, 3, 4, 5, 6, 7}) == 0); double test_mm512_reduce_min_pd(__m512d __W, double ExtraMulOp){ // CHECK-LABEL: @test_mm512_reduce_min_pd( @@ -89,12 +94,14 @@ int test_mm512_reduce_max_epi32(__m512i __W){ // CHECK: call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> %{{.*}}) return _mm512_reduce_max_epi32(__W); } +TEST_CONSTEXPR(_mm512_reduce_max_epi32((__m512i)(__v16si){-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7}) == 7); unsigned int test_mm512_reduce_max_epu32(__m512i __W){ // CHECK-LABEL: @test_mm512_reduce_max_epu32( // CHECK: call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> %{{.*}}) return _mm512_reduce_max_epu32(__W); } +TEST_CONSTEXPR(_mm512_reduce_max_epu32((__m512i)(__v16su){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}) == 15); float test_mm512_reduce_max_ps(__m512 __W){ // CHECK-LABEL: @test_mm512_reduce_max_ps( @@ -107,12 +114,14 @@ int test_mm512_reduce_min_epi32(__m512i __W){ // CHECK: call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> %{{.*}}) return _mm512_reduce_min_epi32(__W); } +TEST_CONSTEXPR(_mm512_reduce_min_epi32((__m512i)(__v16si){-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7}) == -8); unsigned int test_mm512_reduce_min_epu32(__m512i __W){ // CHECK-LABEL: @test_mm512_reduce_min_epu32( // CHECK: call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> %{{.*}}) return _mm512_reduce_min_epu32(__W); } +TEST_CONSTEXPR(_mm512_reduce_min_epu32((__m512i)(__v16su){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}) == 0); float test_mm512_reduce_min_ps(__m512 __W){ // CHECK-LABEL: @test_mm512_reduce_min_ps( diff --git a/clang/test/CodeGen/X86/avx512dq-builtins.c b/clang/test/CodeGen/X86/avx512dq-builtins.c index 1ebd369..1c01695 100644 --- a/clang/test/CodeGen/X86/avx512dq-builtins.c +++ b/clang/test/CodeGen/X86/avx512dq-builtins.c @@ -1,17 +1,21 @@ -// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512dq -emit-llvm -o - -Wall -Werror | FileCheck %s +// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512dq -emit-llvm -o - -Wall -Werror | FileCheck %s +// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512dq -emit-llvm -o - -Wall -Werror | FileCheck %s +// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512dq -emit-llvm -o - -Wall -Werror | FileCheck %s +// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512dq -emit-llvm -o - -Wall -Werror | FileCheck %s #include <immintrin.h> +#include "builtin_test_helpers.h" __mmask8 test_knot_mask8(__mmask8 a) { - // CHECK-LABEL: @test_knot_mask8 + // CHECK-LABEL: test_knot_mask8 // CHECK: [[IN:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[NOT:%.*]] = xor <8 x i1> [[IN]], splat (i1 true) return _knot_mask8(a); } __mmask8 test_kand_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) { - // CHECK-LABEL: @test_kand_mask8 + // CHECK-LABEL: test_kand_mask8 // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RES:%.*]] = and <8 x i1> [[LHS]], [[RHS]] @@ -21,7 +25,7 @@ __mmask8 test_kand_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m } __mmask8 test_kandn_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) { - // CHECK-LABEL: @test_kandn_mask8 + // CHECK-LABEL: test_kandn_mask8 // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[NOT:%.*]] = xor <8 x i1> [[LHS]], splat (i1 true) @@ -32,7 +36,7 @@ __mmask8 test_kandn_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __ } __mmask8 test_kor_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) { - // CHECK-LABEL: @test_kor_mask8 + // CHECK-LABEL: test_kor_mask8 // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RES:%.*]] = or <8 x i1> [[LHS]], [[RHS]] @@ -42,7 +46,7 @@ __mmask8 test_kor_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m5 } __mmask8 test_kxnor_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) { - // CHECK-LABEL: @test_kxnor_mask8 + // CHECK-LABEL: test_kxnor_mask8 // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[NOT:%.*]] = xor <8 x i1> [[LHS]], splat (i1 true) @@ -53,7 +57,7 @@ __mmask8 test_kxnor_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __ } __mmask8 test_kxor_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) { - // CHECK-LABEL: @test_kxor_mask8 + // CHECK-LABEL: test_kxor_mask8 // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RES:%.*]] = xor <8 x i1> [[LHS]], [[RHS]] @@ -63,7 +67,7 @@ __mmask8 test_kxor_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m } unsigned char test_kortestz_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) { - // CHECK-LABEL: @test_kortestz_mask8_u8 + // CHECK-LABEL: test_kortestz_mask8_u8 // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[OR:%.*]] = or <8 x i1> [[LHS]], [[RHS]] @@ -76,7 +80,7 @@ unsigned char test_kortestz_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m5 } unsigned char test_kortestc_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) { - // CHECK-LABEL: @test_kortestc_mask8_u8 + // CHECK-LABEL: test_kortestc_mask8_u8 // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[OR:%.*]] = or <8 x i1> [[LHS]], [[RHS]] @@ -89,7 +93,7 @@ unsigned char test_kortestc_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m5 } unsigned char test_kortest_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, unsigned char *CF) { - // CHECK-LABEL: @test_kortest_mask8_u8 + // CHECK-LABEL: test_kortest_mask8_u8 // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[OR:%.*]] = or <8 x i1> [[LHS]], [[RHS]] @@ -109,7 +113,7 @@ unsigned char test_kortest_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m51 } unsigned char test_ktestz_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) { - // CHECK-LABEL: @test_ktestz_mask8_u8 + // CHECK-LABEL: test_ktestz_mask8_u8 // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestz.b(<8 x i1> [[LHS]], <8 x i1> [[RHS]]) @@ -119,7 +123,7 @@ unsigned char test_ktestz_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512 } unsigned char test_ktestc_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) { - // CHECK-LABEL: @test_ktestc_mask8_u8 + // CHECK-LABEL: test_ktestc_mask8_u8 // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.b(<8 x i1> [[LHS]], <8 x i1> [[RHS]]) @@ -129,7 +133,7 @@ unsigned char test_ktestc_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512 } unsigned char test_ktest_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, unsigned char *CF) { - // CHECK-LABEL: @test_ktest_mask8_u8 + // CHECK-LABEL: test_ktest_mask8_u8 // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.b(<8 x i1> [[LHS]], <8 x i1> [[RHS]]) @@ -143,7 +147,7 @@ unsigned char test_ktest_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i } unsigned char test_ktestz_mask16_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) { - // CHECK-LABEL: @test_ktestz_mask16_u8 + // CHECK-LABEL: test_ktestz_mask16_u8 // CHECK: [[LHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: [[RHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestz.w(<16 x i1> [[LHS]], <16 x i1> [[RHS]]) @@ -153,7 +157,7 @@ unsigned char test_ktestz_mask16_u8(__m512i __A, __m512i __B, __m512i __C, __m51 } unsigned char test_ktestc_mask16_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) { - // CHECK-LABEL: @test_ktestc_mask16_u8 + // CHECK-LABEL: test_ktestc_mask16_u8 // CHECK: [[LHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: [[RHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.w(<16 x i1> [[LHS]], <16 x i1> [[RHS]]) @@ -163,7 +167,7 @@ unsigned char test_ktestc_mask16_u8(__m512i __A, __m512i __B, __m512i __C, __m51 } unsigned char test_ktest_mask16_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, unsigned char *CF) { - // CHECK-LABEL: @test_ktest_mask16_u8 + // CHECK-LABEL: test_ktest_mask16_u8 // CHECK: [[LHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: [[RHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.w(<16 x i1> [[LHS]], <16 x i1> [[RHS]]) @@ -177,7 +181,7 @@ unsigned char test_ktest_mask16_u8(__m512i __A, __m512i __B, __m512i __C, __m512 } __mmask8 test_kadd_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) { - // CHECK-LABEL: @test_kadd_mask8 + // CHECK-LABEL: test_kadd_mask8 // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RES:%.*]] = call <8 x i1> @llvm.x86.avx512.kadd.b(<8 x i1> [[LHS]], <8 x i1> [[RHS]]) @@ -187,7 +191,7 @@ __mmask8 test_kadd_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m } __mmask16 test_kadd_mask16(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) { - // CHECK-LABEL: @test_kadd_mask16 + // CHECK-LABEL: test_kadd_mask16 // CHECK: [[LHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: [[RHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: [[RES:%.*]] = call <16 x i1> @llvm.x86.avx512.kadd.w(<16 x i1> [[LHS]], <16 x i1> [[RHS]]) @@ -197,79 +201,80 @@ __mmask16 test_kadd_mask16(__m512i __A, __m512i __B, __m512i __C, __m512i __D, _ } __mmask8 test_kshiftli_mask8(__m512i A, __m512i B, __m512i C, __m512i D) { - // CHECK-LABEL: @test_kshiftli_mask8 + // CHECK-LABEL: test_kshiftli_mask8 // CHECK: [[VAL:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RES:%.*]] = shufflevector <8 x i1> zeroinitializer, <8 x i1> [[VAL]], <8 x i32> <i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13> return _mm512_mask_cmpneq_epu64_mask(_kshiftli_mask8(_mm512_cmpneq_epu64_mask(A, B), 2), C, D); } __mmask8 test_kshiftri_mask8(__m512i A, __m512i B, __m512i C, __m512i D) { - // CHECK-LABEL: @test_kshiftri_mask8 + // CHECK-LABEL: test_kshiftri_mask8 // CHECK: [[VAL:%.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: [[RES:%.*]] = shufflevector <8 x i1> [[VAL]], <8 x i1> zeroinitializer, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9> return _mm512_mask_cmpneq_epu64_mask(_kshiftri_mask8(_mm512_cmpneq_epu64_mask(A, B), 2), C, D); } unsigned int test_cvtmask8_u32(__m512i A, __m512i B) { - // CHECK-LABEL: @test_cvtmask8_u32 + // CHECK-LABEL: test_cvtmask8_u32 // CHECK: zext i8 %{{.*}} to i32 return _cvtmask8_u32(_mm512_cmpneq_epu64_mask(A, B)); } __mmask8 test_cvtu32_mask8(__m512i A, __m512i B, unsigned int C) { - // CHECK-LABEL: @test_cvtu32_mask8 + // CHECK-LABEL: test_cvtu32_mask8 // CHECK: trunc i32 %{{.*}} to i8 return _mm512_mask_cmpneq_epu64_mask(_cvtu32_mask8(C), A, B); } __mmask8 test_load_mask8(__mmask8 *A, __m512i B, __m512i C) { - // CHECK-LABEL: @test_load_mask8 + // CHECK-LABEL: test_load_mask8 // CHECK: [[LOAD:%.*]] = load i8, ptr %{{.*}} return _mm512_mask_cmpneq_epu64_mask(_load_mask8(A), B, C); } void test_store_mask8(__mmask8 *A, __m512i B, __m512i C) { - // CHECK-LABEL: @test_store_mask8 + // CHECK-LABEL: test_store_mask8 // CHECK: store i8 %{{.*}}, ptr %{{.*}} _store_mask8(A, _mm512_cmpneq_epu64_mask(B, C)); } __m512i test_mm512_mullo_epi64 (__m512i __A, __m512i __B) { - // CHECK-LABEL: @test_mm512_mullo_epi64 + // CHECK-LABEL: test_mm512_mullo_epi64 // CHECK: mul <8 x i64> return (__m512i) _mm512_mullo_epi64(__A, __B); } __m512i test_mm512_mask_mullo_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) { - // CHECK-LABEL: @test_mm512_mask_mullo_epi64 + // CHECK-LABEL: test_mm512_mask_mullo_epi64 // CHECK: mul <8 x i64> %{{.*}}, %{{.*}} // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}} return (__m512i) _mm512_mask_mullo_epi64(__W, __U, __A, __B); } __m512i test_mm512_maskz_mullo_epi64 (__mmask8 __U, __m512i __A, __m512i __B) { - // CHECK-LABEL: @test_mm512_maskz_mullo_epi64 + // CHECK-LABEL: test_mm512_maskz_mullo_epi64 // CHECK: mul <8 x i64> %{{.*}}, %{{.*}} // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}} return (__m512i) _mm512_maskz_mullo_epi64(__U, __A, __B); } __m512d test_mm512_xor_pd (__m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_xor_pd + // CHECK-LABEL: test_mm512_xor_pd // CHECK: xor <8 x i64> return (__m512d) _mm512_xor_pd(__A, __B); } __m512d test_mm512_mask_xor_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_mask_xor_pd + // CHECK-LABEL: test_mm512_mask_xor_pd // CHECK: xor <8 x i64> // CHECK: %[[MASK:.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: select <8 x i1> %[[MASK]], <8 x double> %{{.*}}, <8 x double> %{{.*}} return (__m512d) _mm512_mask_xor_pd(__W, __U, __A, __B); } +TEST_CONSTEXPR(match_m512d(_mm512_xor_pd((__m512d){-4.0, -5.0, +6.0, +7.0, +7.0, +6.0, -5.0, -4.0}, (__m512d){+0.0, -0.0, -0.0, +7.0, +7.0, -0.0, -0.0, +0.0}), -4.0, +5.0, -6.0, +0.0, +0.0, -6.0, +5.0, -4.0)); __m512d test_mm512_maskz_xor_pd (__mmask8 __U, __m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_maskz_xor_pd + // CHECK-LABEL: test_mm512_maskz_xor_pd // CHECK: xor <8 x i64> // CHECK: %[[MASK:.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: select <8 x i1> %[[MASK]], <8 x double> %{{.*}}, <8 x double> %{{.*}} @@ -277,13 +282,14 @@ __m512d test_mm512_maskz_xor_pd (__mmask8 __U, __m512d __A, __m512d __B) { } __m512 test_mm512_xor_ps (__m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_xor_ps + // CHECK-LABEL: test_mm512_xor_ps // CHECK: xor <16 x i32> return (__m512) _mm512_xor_ps(__A, __B); } +TEST_CONSTEXPR(match_m512(_mm512_xor_ps((__m512){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f, -4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m512){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f, +0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), -4.0f, +5.0f, -6.0f, +0.0f, +0.0f, -6.0f, +5.0f, -4.0f, -4.0f, +5.0f, -6.0f, +0.0f, +0.0f, -6.0f, +5.0f, -4.0f)); __m512 test_mm512_mask_xor_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_mask_xor_ps + // CHECK-LABEL: test_mm512_mask_xor_ps // CHECK: xor <16 x i32> // CHECK: %[[MASK:.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: select <16 x i1> %[[MASK]], <16 x float> %{{.*}}, <16 x float> %{{.*}} @@ -291,7 +297,7 @@ __m512 test_mm512_mask_xor_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B } __m512 test_mm512_maskz_xor_ps (__mmask16 __U, __m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_maskz_xor_ps + // CHECK-LABEL: test_mm512_maskz_xor_ps // CHECK: xor <16 x i32> // CHECK: %[[MASK:.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: select <16 x i1> %[[MASK]], <16 x float> %{{.*}}, <16 x float> %{{.*}} @@ -299,13 +305,14 @@ __m512 test_mm512_maskz_xor_ps (__mmask16 __U, __m512 __A, __m512 __B) { } __m512d test_mm512_or_pd (__m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_or_pd + // CHECK-LABEL: test_mm512_or_pd // CHECK: or <8 x i64> return (__m512d) _mm512_or_pd(__A, __B); } +TEST_CONSTEXPR(match_m512d(_mm512_or_pd((__m512d){-4.0, -5.0, +6.0, +7.0, +7.0, +6.0, -5.0, -4.0}, (__m512d){+0.0, -0.0, -0.0, +7.0, +7.0, -0.0, -0.0, +0.0}), -4.0, -5.0, -6.0, +7.0, +7.0, -6.0, -5.0, -4.0)); __m512d test_mm512_mask_or_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_mask_or_pd + // CHECK-LABEL: test_mm512_mask_or_pd // CHECK: or <8 x i64> // CHECK: %[[MASK:.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: select <8 x i1> %[[MASK]], <8 x double> %{{.*}}, <8 x double> %{{.*}} @@ -313,7 +320,7 @@ __m512d test_mm512_mask_or_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d _ } __m512d test_mm512_maskz_or_pd (__mmask8 __U, __m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_maskz_or_pd + // CHECK-LABEL: test_mm512_maskz_or_pd // CHECK: or <8 x i64> // CHECK: %[[MASK:.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: select <8 x i1> %[[MASK]], <8 x double> %{{.*}}, <8 x double> %{{.*}} @@ -321,13 +328,14 @@ __m512d test_mm512_maskz_or_pd (__mmask8 __U, __m512d __A, __m512d __B) { } __m512 test_mm512_or_ps (__m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_or_ps + // CHECK-LABEL: test_mm512_or_ps // CHECK: or <16 x i32> return (__m512) _mm512_or_ps(__A, __B); } +TEST_CONSTEXPR(match_m512(_mm512_or_ps((__m512){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f, -4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m512){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f, +0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), -4.0f, -5.0f, -6.0f, +7.0f, +7.0f, -6.0f, -5.0f, -4.0f, -4.0f, -5.0f, -6.0f, +7.0f, +7.0f, -6.0f, -5.0f, -4.0f)); __m512 test_mm512_mask_or_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_mask_or_ps + // CHECK-LABEL: test_mm512_mask_or_ps // CHECK: or <16 x i32> // CHECK: %[[MASK:.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: select <16 x i1> %[[MASK]], <16 x float> %{{.*}}, <16 x float> %{{.*}} @@ -335,7 +343,7 @@ __m512 test_mm512_mask_or_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) } __m512 test_mm512_maskz_or_ps (__mmask16 __U, __m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_maskz_or_ps + // CHECK-LABEL: test_mm512_maskz_or_ps // CHECK: or <16 x i32> // CHECK: %[[MASK:.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: select <16 x i1> %[[MASK]], <16 x float> %{{.*}}, <16 x float> %{{.*}} @@ -343,13 +351,14 @@ __m512 test_mm512_maskz_or_ps (__mmask16 __U, __m512 __A, __m512 __B) { } __m512d test_mm512_and_pd (__m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_and_pd + // CHECK-LABEL: test_mm512_and_pd // CHECK: and <8 x i64> return (__m512d) _mm512_and_pd(__A, __B); } +TEST_CONSTEXPR(match_m512d(_mm512_and_pd((__m512d){-4.0, -5.0, +6.0, +7.0, +7.0, +6.0, -5.0, -4.0}, (__m512d){+0.0, -0.0, -0.0, +7.0, +7.0, -0.0, -0.0, +0.0}), -0.0, -0.0, +0.0, +7.0, +7.0, +0.0, -0.0, -0.0)); __m512d test_mm512_mask_and_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_mask_and_pd + // CHECK-LABEL: test_mm512_mask_and_pd // CHECK: and <8 x i64> // CHECK: %[[MASK:.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: select <8 x i1> %[[MASK]], <8 x double> %{{.*}}, <8 x double> %{{.*}} @@ -357,7 +366,7 @@ __m512d test_mm512_mask_and_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d } __m512d test_mm512_maskz_and_pd (__mmask8 __U, __m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_maskz_and_pd + // CHECK-LABEL: test_mm512_maskz_and_pd // CHECK: and <8 x i64> // CHECK: %[[MASK:.*]] = bitcast i8 %{{.*}} to <8 x i1> // CHECK: select <8 x i1> %[[MASK]], <8 x double> %{{.*}}, <8 x double> %{{.*}} @@ -365,13 +374,14 @@ __m512d test_mm512_maskz_and_pd (__mmask8 __U, __m512d __A, __m512d __B) { } __m512 test_mm512_and_ps (__m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_and_ps + // CHECK-LABEL: test_mm512_and_ps // CHECK: and <16 x i32> return (__m512) _mm512_and_ps(__A, __B); } +TEST_CONSTEXPR(match_m512(_mm512_and_ps((__m512){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f, -4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m512){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f, +0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), -0.0f, -0.0f, +0.0f, +7.0f, +7.0f, +0.0f, -0.0f, -0.0f, -0.0f, -0.0f, +0.0f, +7.0f, +7.0f, +0.0f, -0.0f, -0.0f)); __m512 test_mm512_mask_and_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_mask_and_ps + // CHECK-LABEL: test_mm512_mask_and_ps // CHECK: and <16 x i32> // CHECK: %[[MASK:.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: select <16 x i1> %[[MASK]], <16 x float> %{{.*}}, <16 x float> %{{.*}} @@ -379,7 +389,7 @@ __m512 test_mm512_mask_and_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B } __m512 test_mm512_maskz_and_ps (__mmask16 __U, __m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_maskz_and_ps + // CHECK-LABEL: test_mm512_maskz_and_ps // CHECK: and <16 x i32> // CHECK: %[[MASK:.*]] = bitcast i16 %{{.*}} to <16 x i1> // CHECK: select <16 x i1> %[[MASK]], <16 x float> %{{.*}}, <16 x float> %{{.*}} @@ -387,14 +397,15 @@ __m512 test_mm512_maskz_and_ps (__mmask16 __U, __m512 __A, __m512 __B) { } __m512d test_mm512_andnot_pd (__m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_andnot_pd + // CHECK-LABEL: test_mm512_andnot_pd // CHECK: xor <8 x i64> %{{.*}}, splat (i64 -1) // CHECK: and <8 x i64> return (__m512d) _mm512_andnot_pd(__A, __B); } +TEST_CONSTEXPR(match_m512d(_mm512_andnot_pd((__m512d){-4.0, -5.0, +6.0, +7.0, +7.0, +6.0, -5.0, -4.0}, (__m512d){+0.0, -0.0, -0.0, +7.0, +7.0, -0.0, -0.0, +0.0}), +0.0, +0.0, +0.0, +0.0, +0.0, +0.0, +0.0, +0.0)); __m512d test_mm512_mask_andnot_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_mask_andnot_pd + // CHECK-LABEL: test_mm512_mask_andnot_pd // CHECK: xor <8 x i64> %{{.*}}, splat (i64 -1) // CHECK: and <8 x i64> %{{.*}}, %{{.*}} // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} @@ -402,7 +413,7 @@ __m512d test_mm512_mask_andnot_pd (__m512d __W, __mmask8 __U, __m512d __A, __m51 } __m512d test_mm512_maskz_andnot_pd (__mmask8 __U, __m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_maskz_andnot_pd + // CHECK-LABEL: test_mm512_maskz_andnot_pd // CHECK: xor <8 x i64> %{{.*}}, splat (i64 -1) // CHECK: and <8 x i64> %{{.*}}, %{{.*}} // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} @@ -410,14 +421,15 @@ __m512d test_mm512_maskz_andnot_pd (__mmask8 __U, __m512d __A, __m512d __B) { } __m512 test_mm512_andnot_ps (__m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_andnot_ps + // CHECK-LABEL: test_mm512_andnot_ps // CHECK: xor <16 x i32> %{{.*}}, splat (i32 -1) // CHECK: and <16 x i32> return (__m512) _mm512_andnot_ps(__A, __B); } +TEST_CONSTEXPR(match_m512(_mm512_andnot_ps((__m512){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f, -4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m512){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f, +0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f)); __m512 test_mm512_mask_andnot_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_mask_andnot_ps + // CHECK-LABEL: test_mm512_mask_andnot_ps // CHECK: xor <16 x i32> %{{.*}}, splat (i32 -1) // CHECK: and <16 x i32> %{{.*}}, %{{.*}} // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}} @@ -425,7 +437,7 @@ __m512 test_mm512_mask_andnot_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 } __m512 test_mm512_maskz_andnot_ps (__mmask16 __U, __m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_maskz_andnot_ps + // CHECK-LABEL: test_mm512_maskz_andnot_ps // CHECK: xor <16 x i32> %{{.*}}, splat (i32 -1) // CHECK: and <16 x i32> %{{.*}}, %{{.*}} // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}} @@ -433,491 +445,491 @@ __m512 test_mm512_maskz_andnot_ps (__mmask16 __U, __m512 __A, __m512 __B) { } __m512i test_mm512_cvtpd_epi64(__m512d __A) { - // CHECK-LABEL: @test_mm512_cvtpd_epi64 + // CHECK-LABEL: test_mm512_cvtpd_epi64 // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.512 return _mm512_cvtpd_epi64(__A); } __m512i test_mm512_mask_cvtpd_epi64(__m512i __W, __mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_mask_cvtpd_epi64 + // CHECK-LABEL: test_mm512_mask_cvtpd_epi64 // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.512 return _mm512_mask_cvtpd_epi64(__W, __U, __A); } __m512i test_mm512_maskz_cvtpd_epi64(__mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_maskz_cvtpd_epi64 + // CHECK-LABEL: test_mm512_maskz_cvtpd_epi64 // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.512 return _mm512_maskz_cvtpd_epi64(__U, __A); } __m512i test_mm512_cvt_roundpd_epi64(__m512d __A) { - // CHECK-LABEL: @test_mm512_cvt_roundpd_epi64 + // CHECK-LABEL: test_mm512_cvt_roundpd_epi64 // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.512 return _mm512_cvt_roundpd_epi64(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512i test_mm512_mask_cvt_roundpd_epi64(__m512i __W, __mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_mask_cvt_roundpd_epi64 + // CHECK-LABEL: test_mm512_mask_cvt_roundpd_epi64 // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.512 return _mm512_mask_cvt_roundpd_epi64(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512i test_mm512_maskz_cvt_roundpd_epi64(__mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_maskz_cvt_roundpd_epi64 + // CHECK-LABEL: test_mm512_maskz_cvt_roundpd_epi64 // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.512 return _mm512_maskz_cvt_roundpd_epi64(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512i test_mm512_cvtpd_epu64(__m512d __A) { - // CHECK-LABEL: @test_mm512_cvtpd_epu64 + // CHECK-LABEL: test_mm512_cvtpd_epu64 // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.512 return _mm512_cvtpd_epu64(__A); } __m512i test_mm512_mask_cvtpd_epu64(__m512i __W, __mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_mask_cvtpd_epu64 + // CHECK-LABEL: test_mm512_mask_cvtpd_epu64 // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.512 return _mm512_mask_cvtpd_epu64(__W, __U, __A); } __m512i test_mm512_maskz_cvtpd_epu64(__mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_maskz_cvtpd_epu64 + // CHECK-LABEL: test_mm512_maskz_cvtpd_epu64 // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.512 return _mm512_maskz_cvtpd_epu64(__U, __A); } __m512i test_mm512_cvt_roundpd_epu64(__m512d __A) { - // CHECK-LABEL: @test_mm512_cvt_roundpd_epu64 + // CHECK-LABEL: test_mm512_cvt_roundpd_epu64 // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.512 return _mm512_cvt_roundpd_epu64(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512i test_mm512_mask_cvt_roundpd_epu64(__m512i __W, __mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_mask_cvt_roundpd_epu64 + // CHECK-LABEL: test_mm512_mask_cvt_roundpd_epu64 // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.512 return _mm512_mask_cvt_roundpd_epu64(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512i test_mm512_maskz_cvt_roundpd_epu64(__mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_maskz_cvt_roundpd_epu64 + // CHECK-LABEL: test_mm512_maskz_cvt_roundpd_epu64 // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.512 return _mm512_maskz_cvt_roundpd_epu64(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512i test_mm512_cvtps_epi64(__m256 __A) { - // CHECK-LABEL: @test_mm512_cvtps_epi64 + // CHECK-LABEL: test_mm512_cvtps_epi64 // CHECK: @llvm.x86.avx512.mask.cvtps2qq.512 return _mm512_cvtps_epi64(__A); } __m512i test_mm512_mask_cvtps_epi64(__m512i __W, __mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_mask_cvtps_epi64 + // CHECK-LABEL: test_mm512_mask_cvtps_epi64 // CHECK: @llvm.x86.avx512.mask.cvtps2qq.512 return _mm512_mask_cvtps_epi64(__W, __U, __A); } __m512i test_mm512_maskz_cvtps_epi64(__mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_maskz_cvtps_epi64 + // CHECK-LABEL: test_mm512_maskz_cvtps_epi64 // CHECK: @llvm.x86.avx512.mask.cvtps2qq.512 return _mm512_maskz_cvtps_epi64(__U, __A); } __m512i test_mm512_cvt_roundps_epi64(__m256 __A) { - // CHECK-LABEL: @test_mm512_cvt_roundps_epi64 + // CHECK-LABEL: test_mm512_cvt_roundps_epi64 // CHECK: @llvm.x86.avx512.mask.cvtps2qq.512 return _mm512_cvt_roundps_epi64(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512i test_mm512_mask_cvt_roundps_epi64(__m512i __W, __mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_mask_cvt_roundps_epi64 + // CHECK-LABEL: test_mm512_mask_cvt_roundps_epi64 // CHECK: @llvm.x86.avx512.mask.cvtps2qq.512 return _mm512_mask_cvt_roundps_epi64(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512i test_mm512_maskz_cvt_roundps_epi64(__mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_maskz_cvt_roundps_epi64 + // CHECK-LABEL: test_mm512_maskz_cvt_roundps_epi64 // CHECK: @llvm.x86.avx512.mask.cvtps2qq.512 return _mm512_maskz_cvt_roundps_epi64(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512i test_mm512_cvtps_epu64(__m256 __A) { - // CHECK-LABEL: @test_mm512_cvtps_epu64 + // CHECK-LABEL: test_mm512_cvtps_epu64 // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.512 return _mm512_cvtps_epu64(__A); } __m512i test_mm512_mask_cvtps_epu64(__m512i __W, __mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_mask_cvtps_epu64 + // CHECK-LABEL: test_mm512_mask_cvtps_epu64 // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.512 return _mm512_mask_cvtps_epu64(__W, __U, __A); } __m512i test_mm512_maskz_cvtps_epu64(__mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_maskz_cvtps_epu64 + // CHECK-LABEL: test_mm512_maskz_cvtps_epu64 // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.512 return _mm512_maskz_cvtps_epu64(__U, __A); } __m512i test_mm512_cvt_roundps_epu64(__m256 __A) { - // CHECK-LABEL: @test_mm512_cvt_roundps_epu64 + // CHECK-LABEL: test_mm512_cvt_roundps_epu64 // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.512 return _mm512_cvt_roundps_epu64(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512i test_mm512_mask_cvt_roundps_epu64(__m512i __W, __mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_mask_cvt_roundps_epu64 + // CHECK-LABEL: test_mm512_mask_cvt_roundps_epu64 // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.512 return _mm512_mask_cvt_roundps_epu64(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512i test_mm512_maskz_cvt_roundps_epu64(__mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_maskz_cvt_roundps_epu64 + // CHECK-LABEL: test_mm512_maskz_cvt_roundps_epu64 // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.512 return _mm512_maskz_cvt_roundps_epu64(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512d test_mm512_cvtepi64_pd(__m512i __A) { - // CHECK-LABEL: @test_mm512_cvtepi64_pd + // CHECK-LABEL: test_mm512_cvtepi64_pd // CHECK: sitofp <8 x i64> %{{.*}} to <8 x double> return _mm512_cvtepi64_pd(__A); } __m512d test_mm512_mask_cvtepi64_pd(__m512d __W, __mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_mask_cvtepi64_pd + // CHECK-LABEL: test_mm512_mask_cvtepi64_pd // CHECK: sitofp <8 x i64> %{{.*}} to <8 x double> // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} return _mm512_mask_cvtepi64_pd(__W, __U, __A); } __m512d test_mm512_maskz_cvtepi64_pd(__mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_maskz_cvtepi64_pd + // CHECK-LABEL: test_mm512_maskz_cvtepi64_pd // CHECK: sitofp <8 x i64> %{{.*}} to <8 x double> // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} return _mm512_maskz_cvtepi64_pd(__U, __A); } __m512d test_mm512_cvt_roundepi64_pd(__m512i __A) { - // CHECK-LABEL: @test_mm512_cvt_roundepi64_pd + // CHECK-LABEL: test_mm512_cvt_roundepi64_pd // CHECK: @llvm.x86.avx512.sitofp.round.v8f64.v8i64 return _mm512_cvt_roundepi64_pd(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512d test_mm512_mask_cvt_roundepi64_pd(__m512d __W, __mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_mask_cvt_roundepi64_pd + // CHECK-LABEL: test_mm512_mask_cvt_roundepi64_pd // CHECK: @llvm.x86.avx512.sitofp.round.v8f64.v8i64 // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} return _mm512_mask_cvt_roundepi64_pd(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512d test_mm512_maskz_cvt_roundepi64_pd(__mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_maskz_cvt_roundepi64_pd + // CHECK-LABEL: test_mm512_maskz_cvt_roundepi64_pd // CHECK: @llvm.x86.avx512.sitofp.round.v8f64.v8i64 // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} return _mm512_maskz_cvt_roundepi64_pd(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m256 test_mm512_cvtepi64_ps(__m512i __A) { - // CHECK-LABEL: @test_mm512_cvtepi64_ps + // CHECK-LABEL: test_mm512_cvtepi64_ps // CHECK: sitofp <8 x i64> %{{.*}} to <8 x float> return _mm512_cvtepi64_ps(__A); } __m256 test_mm512_mask_cvtepi64_ps(__m256 __W, __mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_mask_cvtepi64_ps + // CHECK-LABEL: test_mm512_mask_cvtepi64_ps // CHECK: sitofp <8 x i64> %{{.*}} to <8 x float> // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}} return _mm512_mask_cvtepi64_ps(__W, __U, __A); } __m256 test_mm512_maskz_cvtepi64_ps(__mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_maskz_cvtepi64_ps + // CHECK-LABEL: test_mm512_maskz_cvtepi64_ps // CHECK: sitofp <8 x i64> %{{.*}} to <8 x float> // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}} return _mm512_maskz_cvtepi64_ps(__U, __A); } __m256 test_mm512_cvt_roundepi64_ps(__m512i __A) { - // CHECK-LABEL: @test_mm512_cvt_roundepi64_ps + // CHECK-LABEL: test_mm512_cvt_roundepi64_ps // CHECK: @llvm.x86.avx512.sitofp.round.v8f32.v8i64 return _mm512_cvt_roundepi64_ps(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m256 test_mm512_mask_cvt_roundepi64_ps(__m256 __W, __mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_mask_cvt_roundepi64_ps + // CHECK-LABEL: test_mm512_mask_cvt_roundepi64_ps // CHECK: @llvm.x86.avx512.sitofp.round.v8f32.v8i64 // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}} return _mm512_mask_cvt_roundepi64_ps(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m256 test_mm512_maskz_cvt_roundepi64_ps(__mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_maskz_cvt_roundepi64_ps + // CHECK-LABEL: test_mm512_maskz_cvt_roundepi64_ps // CHECK: @llvm.x86.avx512.sitofp.round.v8f32.v8i64 // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}} return _mm512_maskz_cvt_roundepi64_ps(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512i test_mm512_cvttpd_epi64(__m512d __A) { - // CHECK-LABEL: @test_mm512_cvttpd_epi64 + // CHECK-LABEL: test_mm512_cvttpd_epi64 // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.512 return _mm512_cvttpd_epi64(__A); } __m512i test_mm512_mask_cvttpd_epi64(__m512i __W, __mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_mask_cvttpd_epi64 + // CHECK-LABEL: test_mm512_mask_cvttpd_epi64 // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.512 return _mm512_mask_cvttpd_epi64(__W, __U, __A); } __m512i test_mm512_maskz_cvttpd_epi64(__mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_maskz_cvttpd_epi64 + // CHECK-LABEL: test_mm512_maskz_cvttpd_epi64 // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.512 return _mm512_maskz_cvttpd_epi64(__U, __A); } __m512i test_mm512_cvtt_roundpd_epi64(__m512d __A) { - // CHECK-LABEL: @test_mm512_cvtt_roundpd_epi64 + // CHECK-LABEL: test_mm512_cvtt_roundpd_epi64 // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.512 return _mm512_cvtt_roundpd_epi64(__A, _MM_FROUND_NO_EXC); } __m512i test_mm512_mask_cvtt_roundpd_epi64(__m512i __W, __mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_mask_cvtt_roundpd_epi64 + // CHECK-LABEL: test_mm512_mask_cvtt_roundpd_epi64 // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.512 return _mm512_mask_cvtt_roundpd_epi64(__W, __U, __A, _MM_FROUND_NO_EXC); } __m512i test_mm512_maskz_cvtt_roundpd_epi64(__mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_maskz_cvtt_roundpd_epi64 + // CHECK-LABEL: test_mm512_maskz_cvtt_roundpd_epi64 // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.512 return _mm512_maskz_cvtt_roundpd_epi64(__U, __A, _MM_FROUND_NO_EXC); } __m512i test_mm512_cvttpd_epu64(__m512d __A) { - // CHECK-LABEL: @test_mm512_cvttpd_epu64 + // CHECK-LABEL: test_mm512_cvttpd_epu64 // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.512 return _mm512_cvttpd_epu64(__A); } __m512i test_mm512_mask_cvttpd_epu64(__m512i __W, __mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_mask_cvttpd_epu64 + // CHECK-LABEL: test_mm512_mask_cvttpd_epu64 // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.512 return _mm512_mask_cvttpd_epu64(__W, __U, __A); } __m512i test_mm512_maskz_cvttpd_epu64(__mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_maskz_cvttpd_epu64 + // CHECK-LABEL: test_mm512_maskz_cvttpd_epu64 // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.512 return _mm512_maskz_cvttpd_epu64(__U, __A); } __m512i test_mm512_cvtt_roundpd_epu64(__m512d __A) { - // CHECK-LABEL: @test_mm512_cvtt_roundpd_epu64 + // CHECK-LABEL: test_mm512_cvtt_roundpd_epu64 // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.512 return _mm512_cvtt_roundpd_epu64(__A, _MM_FROUND_NO_EXC); } __m512i test_mm512_mask_cvtt_roundpd_epu64(__m512i __W, __mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_mask_cvtt_roundpd_epu64 + // CHECK-LABEL: test_mm512_mask_cvtt_roundpd_epu64 // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.512 return _mm512_mask_cvtt_roundpd_epu64(__W, __U, __A, _MM_FROUND_NO_EXC); } __m512i test_mm512_maskz_cvtt_roundpd_epu64(__mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_maskz_cvtt_roundpd_epu64 + // CHECK-LABEL: test_mm512_maskz_cvtt_roundpd_epu64 // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.512 return _mm512_maskz_cvtt_roundpd_epu64(__U, __A, _MM_FROUND_NO_EXC); } __m512i test_mm512_cvttps_epi64(__m256 __A) { - // CHECK-LABEL: @test_mm512_cvttps_epi64 + // CHECK-LABEL: test_mm512_cvttps_epi64 // CHECK: @llvm.x86.avx512.mask.cvttps2qq.512 return _mm512_cvttps_epi64(__A); } __m512i test_mm512_mask_cvttps_epi64(__m512i __W, __mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_mask_cvttps_epi64 + // CHECK-LABEL: test_mm512_mask_cvttps_epi64 // CHECK: @llvm.x86.avx512.mask.cvttps2qq.512 return _mm512_mask_cvttps_epi64(__W, __U, __A); } __m512i test_mm512_maskz_cvttps_epi64(__mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_maskz_cvttps_epi64 + // CHECK-LABEL: test_mm512_maskz_cvttps_epi64 // CHECK: @llvm.x86.avx512.mask.cvttps2qq.512 return _mm512_maskz_cvttps_epi64(__U, __A); } __m512i test_mm512_cvtt_roundps_epi64(__m256 __A) { - // CHECK-LABEL: @test_mm512_cvtt_roundps_epi64 + // CHECK-LABEL: test_mm512_cvtt_roundps_epi64 // CHECK: @llvm.x86.avx512.mask.cvttps2qq.512 return _mm512_cvtt_roundps_epi64(__A, _MM_FROUND_NO_EXC); } __m512i test_mm512_mask_cvtt_roundps_epi64(__m512i __W, __mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_mask_cvtt_roundps_epi64 + // CHECK-LABEL: test_mm512_mask_cvtt_roundps_epi64 // CHECK: @llvm.x86.avx512.mask.cvttps2qq.512 return _mm512_mask_cvtt_roundps_epi64(__W, __U, __A, _MM_FROUND_NO_EXC); } __m512i test_mm512_maskz_cvtt_roundps_epi64(__mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_maskz_cvtt_roundps_epi64 + // CHECK-LABEL: test_mm512_maskz_cvtt_roundps_epi64 // CHECK: @llvm.x86.avx512.mask.cvttps2qq.512 return _mm512_maskz_cvtt_roundps_epi64(__U, __A, _MM_FROUND_NO_EXC); } __m512i test_mm512_cvttps_epu64(__m256 __A) { - // CHECK-LABEL: @test_mm512_cvttps_epu64 + // CHECK-LABEL: test_mm512_cvttps_epu64 // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.512 return _mm512_cvttps_epu64(__A); } __m512i test_mm512_mask_cvttps_epu64(__m512i __W, __mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_mask_cvttps_epu64 + // CHECK-LABEL: test_mm512_mask_cvttps_epu64 // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.512 return _mm512_mask_cvttps_epu64(__W, __U, __A); } __m512i test_mm512_maskz_cvttps_epu64(__mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_maskz_cvttps_epu64 + // CHECK-LABEL: test_mm512_maskz_cvttps_epu64 // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.512 return _mm512_maskz_cvttps_epu64(__U, __A); } __m512i test_mm512_cvtt_roundps_epu64(__m256 __A) { - // CHECK-LABEL: @test_mm512_cvtt_roundps_epu64 + // CHECK-LABEL: test_mm512_cvtt_roundps_epu64 // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.512 return _mm512_cvtt_roundps_epu64(__A, _MM_FROUND_NO_EXC); } __m512i test_mm512_mask_cvtt_roundps_epu64(__m512i __W, __mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_mask_cvtt_roundps_epu64 + // CHECK-LABEL: test_mm512_mask_cvtt_roundps_epu64 // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.512 return _mm512_mask_cvtt_roundps_epu64(__W, __U, __A, _MM_FROUND_NO_EXC); } __m512i test_mm512_maskz_cvtt_roundps_epu64(__mmask8 __U, __m256 __A) { - // CHECK-LABEL: @test_mm512_maskz_cvtt_roundps_epu64 + // CHECK-LABEL: test_mm512_maskz_cvtt_roundps_epu64 // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.512 return _mm512_maskz_cvtt_roundps_epu64(__U, __A, _MM_FROUND_NO_EXC); } __m512d test_mm512_cvtepu64_pd(__m512i __A) { - // CHECK-LABEL: @test_mm512_cvtepu64_pd + // CHECK-LABEL: test_mm512_cvtepu64_pd // CHECK: uitofp <8 x i64> %{{.*}} to <8 x double> return _mm512_cvtepu64_pd(__A); } __m512d test_mm512_mask_cvtepu64_pd(__m512d __W, __mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_mask_cvtepu64_pd + // CHECK-LABEL: test_mm512_mask_cvtepu64_pd // CHECK: uitofp <8 x i64> %{{.*}} to <8 x double> // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} return _mm512_mask_cvtepu64_pd(__W, __U, __A); } __m512d test_mm512_maskz_cvtepu64_pd(__mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_maskz_cvtepu64_pd + // CHECK-LABEL: test_mm512_maskz_cvtepu64_pd // CHECK: uitofp <8 x i64> %{{.*}} to <8 x double> // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} return _mm512_maskz_cvtepu64_pd(__U, __A); } __m512d test_mm512_cvt_roundepu64_pd(__m512i __A) { - // CHECK-LABEL: @test_mm512_cvt_roundepu64_pd + // CHECK-LABEL: test_mm512_cvt_roundepu64_pd // CHECK: @llvm.x86.avx512.uitofp.round.v8f64.v8i64 return _mm512_cvt_roundepu64_pd(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512d test_mm512_mask_cvt_roundepu64_pd(__m512d __W, __mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_mask_cvt_roundepu64_pd + // CHECK-LABEL: test_mm512_mask_cvt_roundepu64_pd // CHECK: @llvm.x86.avx512.uitofp.round.v8f64.v8i64 // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} return _mm512_mask_cvt_roundepu64_pd(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512d test_mm512_maskz_cvt_roundepu64_pd(__mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_maskz_cvt_roundepu64_pd + // CHECK-LABEL: test_mm512_maskz_cvt_roundepu64_pd // CHECK: @llvm.x86.avx512.uitofp.round.v8f64.v8i64 // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} return _mm512_maskz_cvt_roundepu64_pd(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m256 test_mm512_cvtepu64_ps(__m512i __A) { - // CHECK-LABEL: @test_mm512_cvtepu64_ps + // CHECK-LABEL: test_mm512_cvtepu64_ps // CHECK: uitofp <8 x i64> %{{.*}} to <8 x float> return _mm512_cvtepu64_ps(__A); } __m256 test_mm512_mask_cvtepu64_ps(__m256 __W, __mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_mask_cvtepu64_ps + // CHECK-LABEL: test_mm512_mask_cvtepu64_ps // CHECK: uitofp <8 x i64> %{{.*}} to <8 x float> // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}} return _mm512_mask_cvtepu64_ps(__W, __U, __A); } __m256 test_mm512_maskz_cvtepu64_ps(__mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_maskz_cvtepu64_ps + // CHECK-LABEL: test_mm512_maskz_cvtepu64_ps // CHECK: uitofp <8 x i64> %{{.*}} to <8 x float> // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}} return _mm512_maskz_cvtepu64_ps(__U, __A); } __m256 test_mm512_cvt_roundepu64_ps(__m512i __A) { - // CHECK-LABEL: @test_mm512_cvt_roundepu64_ps + // CHECK-LABEL: test_mm512_cvt_roundepu64_ps // CHECK: @llvm.x86.avx512.uitofp.round.v8f32.v8i64 return _mm512_cvt_roundepu64_ps(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m256 test_mm512_mask_cvt_roundepu64_ps(__m256 __W, __mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_mask_cvt_roundepu64_ps + // CHECK-LABEL: test_mm512_mask_cvt_roundepu64_ps // CHECK: @llvm.x86.avx512.uitofp.round.v8f32.v8i64 // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}} return _mm512_mask_cvt_roundepu64_ps(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m256 test_mm512_maskz_cvt_roundepu64_ps(__mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_maskz_cvt_roundepu64_ps + // CHECK-LABEL: test_mm512_maskz_cvt_roundepu64_ps // CHECK: @llvm.x86.avx512.uitofp.round.v8f32.v8i64 // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}} return _mm512_maskz_cvt_roundepu64_ps(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); } __m512d test_mm512_range_pd(__m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_range_pd + // CHECK-LABEL: test_mm512_range_pd // CHECK: @llvm.x86.avx512.mask.range.pd.512 return _mm512_range_pd(__A, __B, 4); } __m512d test_mm512_mask_range_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_mask_range_pd + // CHECK-LABEL: test_mm512_mask_range_pd // CHECK: @llvm.x86.avx512.mask.range.pd.512 return _mm512_mask_range_pd(__W, __U, __A, __B, 4); } __m512d test_mm512_maskz_range_pd(__mmask8 __U, __m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_maskz_range_pd + // CHECK-LABEL: test_mm512_maskz_range_pd // CHECK: @llvm.x86.avx512.mask.range.pd.512 return _mm512_maskz_range_pd(__U, __A, __B, 4); } __m512d test_mm512_range_round_pd(__m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_range_round_pd + // CHECK-LABEL: test_mm512_range_round_pd // CHECK: @llvm.x86.avx512.mask.range.pd.512 return _mm512_range_round_pd(__A, __B, 4, 8); } __m512d test_mm512_mask_range_round_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_mask_range_round_pd + // CHECK-LABEL: test_mm512_mask_range_round_pd // CHECK: @llvm.x86.avx512.mask.range.pd.512 return _mm512_mask_range_round_pd(__W, __U, __A, __B, 4, 8); } __m512d test_mm512_maskz_range_round_pd(__mmask8 __U, __m512d __A, __m512d __B) { - // CHECK-LABEL: @test_mm512_maskz_range_round_pd + // CHECK-LABEL: test_mm512_maskz_range_round_pd // CHECK: @llvm.x86.avx512.mask.range.pd.512 return _mm512_maskz_range_round_pd(__U, __A, __B, 4, 8); } __m128d test_mm512_range_round_sd(__m128d __A, __m128d __B) { - // CHECK-LABEL: @test_mm512_range_round_sd + // CHECK-LABEL: test_mm512_range_round_sd // CHECK: @llvm.x86.avx512.mask.range.sd return _mm_range_round_sd(__A, __B, 4, 8); } @@ -929,31 +941,31 @@ __m128d test_mm512_mask_range_round_sd(__m128d __W, __mmask8 __U, __m128d __A, _ } __m128d test_mm512_maskz_range_round_sd(__mmask8 __U, __m128d __A, __m128d __B) { - // CHECK-LABEL: @test_mm512_maskz_range_round_sd + // CHECK-LABEL: test_mm512_maskz_range_round_sd // CHECK: @llvm.x86.avx512.mask.range.sd return _mm_maskz_range_round_sd(__U, __A, __B, 4, 8); } __m128 test_mm512_range_round_ss(__m128 __A, __m128 __B) { - // CHECK-LABEL: @test_mm512_range_round_ss + // CHECK-LABEL: test_mm512_range_round_ss // CHECK: @llvm.x86.avx512.mask.range.ss return _mm_range_round_ss(__A, __B, 4, 8); } __m128 test_mm512_mask_range_round_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) { - // CHECK-LABEL: @test_mm512_mask_range_round_ss + // CHECK-LABEL: test_mm512_mask_range_round_ss // CHECK: @llvm.x86.avx512.mask.range.ss return _mm_mask_range_round_ss(__W, __U, __A, __B, 4, 8); } __m128 test_mm512_maskz_range_round_ss(__mmask8 __U, __m128 __A, __m128 __B) { - // CHECK-LABEL: @test_mm512_maskz_range_round_ss + // CHECK-LABEL: test_mm512_maskz_range_round_ss // CHECK: @llvm.x86.avx512.mask.range.ss return _mm_maskz_range_round_ss(__U, __A, __B, 4, 8); } __m128d test_mm_range_sd(__m128d __A, __m128d __B) { - // CHECK-LABEL: @test_mm_range_sd + // CHECK-LABEL: test_mm_range_sd // CHECK: @llvm.x86.avx512.mask.range.sd return _mm_range_sd(__A, __B, 4); } @@ -965,558 +977,558 @@ __m128d test_mm_mask_range_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __ } __m128d test_mm_maskz_range_sd(__mmask8 __U, __m128d __A, __m128d __B) { - // CHECK-LABEL: @test_mm_maskz_range_sd + // CHECK-LABEL: test_mm_maskz_range_sd // CHECK: @llvm.x86.avx512.mask.range.sd return _mm_maskz_range_sd(__U, __A, __B, 4); } __m128 test_mm_range_ss(__m128 __A, __m128 __B) { - // CHECK-LABEL: @test_mm_range_ss + // CHECK-LABEL: test_mm_range_ss // CHECK: @llvm.x86.avx512.mask.range.ss return _mm_range_ss(__A, __B, 4); } __m128 test_mm_mask_range_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) { - // CHECK-LABEL: @test_mm_mask_range_ss + // CHECK-LABEL: test_mm_mask_range_ss // CHECK: @llvm.x86.avx512.mask.range.ss return _mm_mask_range_ss(__W, __U, __A, __B, 4); } __m128 test_mm_maskz_range_ss(__mmask8 __U, __m128 __A, __m128 __B) { - // CHECK-LABEL: @test_mm_maskz_range_ss + // CHECK-LABEL: test_mm_maskz_range_ss // CHECK: @llvm.x86.avx512.mask.range.ss return _mm_maskz_range_ss(__U, __A, __B, 4); } __m512 test_mm512_range_ps(__m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_range_ps + // CHECK-LABEL: test_mm512_range_ps // CHECK: @llvm.x86.avx512.mask.range.ps.512 return _mm512_range_ps(__A, __B, 4); } __m512 test_mm512_mask_range_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_mask_range_ps + // CHECK-LABEL: test_mm512_mask_range_ps // CHECK: @llvm.x86.avx512.mask.range.ps.512 return _mm512_mask_range_ps(__W, __U, __A, __B, 4); } __m512 test_mm512_maskz_range_ps(__mmask16 __U, __m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_maskz_range_ps + // CHECK-LABEL: test_mm512_maskz_range_ps // CHECK: @llvm.x86.avx512.mask.range.ps.512 return _mm512_maskz_range_ps(__U, __A, __B, 4); } __m512 test_mm512_range_round_ps(__m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_range_round_ps + // CHECK-LABEL: test_mm512_range_round_ps // CHECK: @llvm.x86.avx512.mask.range.ps.512 return _mm512_range_round_ps(__A, __B, 4, 8); } __m512 test_mm512_mask_range_round_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_mask_range_round_ps + // CHECK-LABEL: test_mm512_mask_range_round_ps // CHECK: @llvm.x86.avx512.mask.range.ps.512 return _mm512_mask_range_round_ps(__W, __U, __A, __B, 4, 8); } __m512 test_mm512_maskz_range_round_ps(__mmask16 __U, __m512 __A, __m512 __B) { - // CHECK-LABEL: @test_mm512_maskz_range_round_ps + // CHECK-LABEL: test_mm512_maskz_range_round_ps // CHECK: @llvm.x86.avx512.mask.range.ps.512 return _mm512_maskz_range_round_ps(__U, __A, __B, 4, 8); } __m512d test_mm512_reduce_pd(__m512d __A) { - // CHECK-LABEL: @test_mm512_reduce_pd + // CHECK-LABEL: test_mm512_reduce_pd // CHECK: @llvm.x86.avx512.mask.reduce.pd.512 return _mm512_reduce_pd(__A, 4); } __m512d test_mm512_mask_reduce_pd(__m512d __W, __mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_mask_reduce_pd + // CHECK-LABEL: test_mm512_mask_reduce_pd // CHECK: @llvm.x86.avx512.mask.reduce.pd.512 return _mm512_mask_reduce_pd(__W, __U, __A, 4); } __m512d test_mm512_maskz_reduce_pd(__mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_maskz_reduce_pd + // CHECK-LABEL: test_mm512_maskz_reduce_pd // CHECK: @llvm.x86.avx512.mask.reduce.pd.512 return _mm512_maskz_reduce_pd(__U, __A, 4); } __m512 test_mm512_reduce_ps(__m512 __A) { - // CHECK-LABEL: @test_mm512_reduce_ps + // CHECK-LABEL: test_mm512_reduce_ps // CHECK: @llvm.x86.avx512.mask.reduce.ps.512 return _mm512_reduce_ps(__A, 4); } __m512 test_mm512_mask_reduce_ps(__m512 __W, __mmask16 __U, __m512 __A) { - // CHECK-LABEL: @test_mm512_mask_reduce_ps + // CHECK-LABEL: test_mm512_mask_reduce_ps // CHECK: @llvm.x86.avx512.mask.reduce.ps.512 return _mm512_mask_reduce_ps(__W, __U, __A, 4); } __m512 test_mm512_maskz_reduce_ps(__mmask16 __U, __m512 __A) { - // CHECK-LABEL: @test_mm512_maskz_reduce_ps + // CHECK-LABEL: test_mm512_maskz_reduce_ps // CHECK: @llvm.x86.avx512.mask.reduce.ps.512 return _mm512_maskz_reduce_ps(__U, __A, 4); } __m512d test_mm512_reduce_round_pd(__m512d __A) { - // CHECK-LABEL: @test_mm512_reduce_round_pd + // CHECK-LABEL: test_mm512_reduce_round_pd // CHECK: @llvm.x86.avx512.mask.reduce.pd.512 return _mm512_reduce_round_pd(__A, 4, 8); } __m512d test_mm512_mask_reduce_round_pd(__m512d __W, __mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_mask_reduce_round_pd + // CHECK-LABEL: test_mm512_mask_reduce_round_pd // CHECK: @llvm.x86.avx512.mask.reduce.pd.512 return _mm512_mask_reduce_round_pd(__W, __U, __A, 4, 8); } __m512d test_mm512_maskz_reduce_round_pd(__mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_maskz_reduce_round_pd + // CHECK-LABEL: test_mm512_maskz_reduce_round_pd // CHECK: @llvm.x86.avx512.mask.reduce.pd.512 return _mm512_maskz_reduce_round_pd(__U, __A, 4, 8); } __m512 test_mm512_reduce_round_ps(__m512 __A) { - // CHECK-LABEL: @test_mm512_reduce_round_ps + // CHECK-LABEL: test_mm512_reduce_round_ps // CHECK: @llvm.x86.avx512.mask.reduce.ps.512 return _mm512_reduce_round_ps(__A, 4, 8); } __m512 test_mm512_mask_reduce_round_ps(__m512 __W, __mmask16 __U, __m512 __A) { - // CHECK-LABEL: @test_mm512_mask_reduce_round_ps + // CHECK-LABEL: test_mm512_mask_reduce_round_ps // CHECK: @llvm.x86.avx512.mask.reduce.ps.512 return _mm512_mask_reduce_round_ps(__W, __U, __A, 4, 8); } __m512 test_mm512_maskz_reduce_round_ps(__mmask16 __U, __m512 __A) { - // CHECK-LABEL: @test_mm512_maskz_reduce_round_ps + // CHECK-LABEL: test_mm512_maskz_reduce_round_ps // CHECK: @llvm.x86.avx512.mask.reduce.ps.512 return _mm512_maskz_reduce_round_ps(__U, __A, 4, 8); } __m128 test_mm_reduce_ss(__m128 __A, __m128 __B) { - // CHECK-LABEL: @test_mm_reduce_ss + // CHECK-LABEL: test_mm_reduce_ss // CHECK: @llvm.x86.avx512.mask.reduce.ss return _mm_reduce_ss(__A, __B, 4); } __m128 test_mm_mask_reduce_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) { - // CHECK-LABEL: @test_mm_mask_reduce_ss + // CHECK-LABEL: test_mm_mask_reduce_ss // CHECK: @llvm.x86.avx512.mask.reduce.ss return _mm_mask_reduce_ss(__W, __U, __A, __B, 4); } __m128 test_mm_maskz_reduce_ss(__mmask8 __U, __m128 __A, __m128 __B) { - // CHECK-LABEL: @test_mm_maskz_reduce_ss + // CHECK-LABEL: test_mm_maskz_reduce_ss // CHECK: @llvm.x86.avx512.mask.reduce.ss return _mm_maskz_reduce_ss(__U, __A, __B, 4); } __m128 test_mm_reduce_round_ss(__m128 __A, __m128 __B) { - // CHECK-LABEL: @test_mm_reduce_round_ss + // CHECK-LABEL: test_mm_reduce_round_ss // CHECK: @llvm.x86.avx512.mask.reduce.ss return _mm_reduce_round_ss(__A, __B, 4, 8); } __m128 test_mm_mask_reduce_round_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) { - // CHECK-LABEL: @test_mm_mask_reduce_round_ss + // CHECK-LABEL: test_mm_mask_reduce_round_ss // CHECK: @llvm.x86.avx512.mask.reduce.ss return _mm_mask_reduce_round_ss(__W, __U, __A, __B, 4, 8); } __m128 test_mm_maskz_reduce_round_ss(__mmask8 __U, __m128 __A, __m128 __B) { - // CHECK-LABEL: @test_mm_maskz_reduce_round_ss + // CHECK-LABEL: test_mm_maskz_reduce_round_ss // CHECK: @llvm.x86.avx512.mask.reduce.ss return _mm_maskz_reduce_round_ss(__U, __A, __B, 4, 8); } __m128d test_mm_reduce_sd(__m128d __A, __m128d __B) { - // CHECK-LABEL: @test_mm_reduce_sd + // CHECK-LABEL: test_mm_reduce_sd // CHECK: @llvm.x86.avx512.mask.reduce.sd return _mm_reduce_sd(__A, __B, 4); } __m128d test_mm_mask_reduce_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) { - // CHECK-LABEL: @test_mm_mask_reduce_sd + // CHECK-LABEL: test_mm_mask_reduce_sd // CHECK: @llvm.x86.avx512.mask.reduce.sd return _mm_mask_reduce_sd(__W, __U, __A, __B, 4); } __m128d test_mm_maskz_reduce_sd(__mmask8 __U, __m128d __A, __m128d __B) { - // CHECK-LABEL: @test_mm_maskz_reduce_sd + // CHECK-LABEL: test_mm_maskz_reduce_sd // CHECK: @llvm.x86.avx512.mask.reduce.sd return _mm_maskz_reduce_sd(__U, __A, __B, 4); } __m128d test_mm_reduce_round_sd(__m128d __A, __m128d __B) { - // CHECK-LABEL: @test_mm_reduce_round_sd + // CHECK-LABEL: test_mm_reduce_round_sd // CHECK: @llvm.x86.avx512.mask.reduce.sd return _mm_reduce_round_sd(__A, __B, 4, 8); } __m128d test_mm_mask_reduce_round_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) { - // CHECK-LABEL: @test_mm_mask_reduce_round_sd + // CHECK-LABEL: test_mm_mask_reduce_round_sd // CHECK: @llvm.x86.avx512.mask.reduce.sd return _mm_mask_reduce_round_sd(__W, __U, __A, __B, 4, 8); } __m128d test_mm_maskz_reduce_round_sd(__mmask8 __U, __m128d __A, __m128d __B) { - // CHECK-LABEL: @test_mm_maskz_reduce_round_sd + // CHECK-LABEL: test_mm_maskz_reduce_round_sd // CHECK: @llvm.x86.avx512.mask.reduce.sd return _mm_maskz_reduce_round_sd(__U, __A, __B, 4, 8); } __mmask16 test_mm512_movepi32_mask(__m512i __A) { - // CHECK-LABEL: @test_mm512_movepi32_mask + // CHECK-LABEL: test_mm512_movepi32_mask // CHECK: [[CMP:%.*]] = icmp slt <16 x i32> %{{.*}}, zeroinitializer return _mm512_movepi32_mask(__A); } __m512i test_mm512_movm_epi32(__mmask16 __A) { - // CHECK-LABEL: @test_mm512_movm_epi32 + // CHECK-LABEL: test_mm512_movm_epi32 // CHECK: %{{.*}} = bitcast i16 %{{.*}} to <16 x i1> // CHECK: %vpmovm2.i = sext <16 x i1> %{{.*}} to <16 x i32> return _mm512_movm_epi32(__A); } __m512i test_mm512_movm_epi64(__mmask8 __A) { - // CHECK-LABEL: @test_mm512_movm_epi64 + // CHECK-LABEL: test_mm512_movm_epi64 // CHECK: %{{.*}} = bitcast i8 %{{.*}} to <8 x i1> // CHECK: %vpmovm2.i = sext <8 x i1> %{{.*}} to <8 x i64> return _mm512_movm_epi64(__A); } __mmask8 test_mm512_movepi64_mask(__m512i __A) { - // CHECK-LABEL: @test_mm512_movepi64_mask + // CHECK-LABEL: test_mm512_movepi64_mask // CHECK: [[CMP:%.*]] = icmp slt <8 x i64> %{{.*}}, zeroinitializer return _mm512_movepi64_mask(__A); } __m512 test_mm512_broadcast_f32x2(__m128 __A) { - // CHECK-LABEL: @test_mm512_broadcast_f32x2 + // CHECK-LABEL: test_mm512_broadcast_f32x2 // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1> return _mm512_broadcast_f32x2(__A); } __m512 test_mm512_mask_broadcast_f32x2(__m512 __O, __mmask16 __M, __m128 __A) { - // CHECK-LABEL: @test_mm512_mask_broadcast_f32x2 + // CHECK-LABEL: test_mm512_mask_broadcast_f32x2 // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1> // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}} return _mm512_mask_broadcast_f32x2(__O, __M, __A); } __m512 test_mm512_maskz_broadcast_f32x2(__mmask16 __M, __m128 __A) { - // CHECK-LABEL: @test_mm512_maskz_broadcast_f32x2 + // CHECK-LABEL: test_mm512_maskz_broadcast_f32x2 // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1> // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}} return _mm512_maskz_broadcast_f32x2(__M, __A); } __m512 test_mm512_broadcast_f32x8(float const* __A) { - // CHECK-LABEL: @test_mm512_broadcast_f32x8 + // CHECK-LABEL: test_mm512_broadcast_f32x8 // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> return _mm512_broadcast_f32x8(_mm256_loadu_ps(__A)); } __m512 test_mm512_mask_broadcast_f32x8(__m512 __O, __mmask16 __M, float const* __A) { - // CHECK-LABEL: @test_mm512_mask_broadcast_f32x8 + // CHECK-LABEL: test_mm512_mask_broadcast_f32x8 // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}} return _mm512_mask_broadcast_f32x8(__O, __M, _mm256_loadu_ps(__A)); } __m512 test_mm512_maskz_broadcast_f32x8(__mmask16 __M, float const* __A) { - // CHECK-LABEL: @test_mm512_maskz_broadcast_f32x8 + // CHECK-LABEL: test_mm512_maskz_broadcast_f32x8 // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}} return _mm512_maskz_broadcast_f32x8(__M, _mm256_loadu_ps(__A)); } __m512d test_mm512_broadcast_f64x2(double const* __A) { - // CHECK-LABEL: @test_mm512_broadcast_f64x2 + // CHECK-LABEL: test_mm512_broadcast_f64x2 // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1> return _mm512_broadcast_f64x2(_mm_loadu_pd(__A)); } __m512d test_mm512_mask_broadcast_f64x2(__m512d __O, __mmask8 __M, double const* __A) { - // CHECK-LABEL: @test_mm512_mask_broadcast_f64x2 + // CHECK-LABEL: test_mm512_mask_broadcast_f64x2 // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1> // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} return _mm512_mask_broadcast_f64x2(__O, __M, _mm_loadu_pd(__A)); } __m512d test_mm512_maskz_broadcast_f64x2(__mmask8 __M, double const* __A) { - // CHECK-LABEL: @test_mm512_maskz_broadcast_f64x2 + // CHECK-LABEL: test_mm512_maskz_broadcast_f64x2 // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1> // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} return _mm512_maskz_broadcast_f64x2(__M, _mm_loadu_pd(__A)); } __m512i test_mm512_broadcast_i32x2(__m128i __A) { - // CHECK-LABEL: @test_mm512_broadcast_i32x2 + // CHECK-LABEL: test_mm512_broadcast_i32x2 // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1> return _mm512_broadcast_i32x2(__A); } __m512i test_mm512_mask_broadcast_i32x2(__m512i __O, __mmask16 __M, __m128i __A) { - // CHECK-LABEL: @test_mm512_mask_broadcast_i32x2 + // CHECK-LABEL: test_mm512_mask_broadcast_i32x2 // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1> // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}} return _mm512_mask_broadcast_i32x2(__O, __M, __A); } __m512i test_mm512_maskz_broadcast_i32x2(__mmask16 __M, __m128i __A) { - // CHECK-LABEL: @test_mm512_maskz_broadcast_i32x2 + // CHECK-LABEL: test_mm512_maskz_broadcast_i32x2 // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1> // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}} return _mm512_maskz_broadcast_i32x2(__M, __A); } __m512i test_mm512_broadcast_i32x8(__m256i const* __A) { - // CHECK-LABEL: @test_mm512_broadcast_i32x8 + // CHECK-LABEL: test_mm512_broadcast_i32x8 // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> return _mm512_broadcast_i32x8(_mm256_loadu_si256(__A)); } __m512i test_mm512_mask_broadcast_i32x8(__m512i __O, __mmask16 __M, __m256i const* __A) { - // CHECK-LABEL: @test_mm512_mask_broadcast_i32x8 + // CHECK-LABEL: test_mm512_mask_broadcast_i32x8 // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}} return _mm512_mask_broadcast_i32x8(__O, __M, _mm256_loadu_si256(__A)); } __m512i test_mm512_maskz_broadcast_i32x8(__mmask16 __M, __m256i const* __A) { - // CHECK-LABEL: @test_mm512_maskz_broadcast_i32x8 + // CHECK-LABEL: test_mm512_maskz_broadcast_i32x8 // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}} return _mm512_maskz_broadcast_i32x8(__M, _mm256_loadu_si256(__A)); } __m512i test_mm512_broadcast_i64x2(__m128i const* __A) { - // CHECK-LABEL: @test_mm512_broadcast_i64x2 + // CHECK-LABEL: test_mm512_broadcast_i64x2 // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1> return _mm512_broadcast_i64x2(_mm_loadu_si128(__A)); } __m512i test_mm512_mask_broadcast_i64x2(__m512i __O, __mmask8 __M, __m128i const* __A) { - // CHECK-LABEL: @test_mm512_mask_broadcast_i64x2 + // CHECK-LABEL: test_mm512_mask_broadcast_i64x2 // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1> // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}} return _mm512_mask_broadcast_i64x2(__O, __M, _mm_loadu_si128(__A)); } __m512i test_mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i const* __A) { - // CHECK-LABEL: @test_mm512_maskz_broadcast_i64x2 + // CHECK-LABEL: test_mm512_maskz_broadcast_i64x2 // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1> // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}} return _mm512_maskz_broadcast_i64x2(__M, _mm_loadu_si128(__A)); } __m256 test_mm512_extractf32x8_ps(__m512 __A) { - // CHECK-LABEL: @test_mm512_extractf32x8_ps + // CHECK-LABEL: test_mm512_extractf32x8_ps // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> return _mm512_extractf32x8_ps(__A, 1); } __m256 test_mm512_mask_extractf32x8_ps(__m256 __W, __mmask8 __U, __m512 __A) { - // CHECK-LABEL: @test_mm512_mask_extractf32x8_ps + // CHECK-LABEL: test_mm512_mask_extractf32x8_ps // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}} return _mm512_mask_extractf32x8_ps(__W, __U, __A, 1); } __m256 test_mm512_maskz_extractf32x8_ps(__mmask8 __U, __m512 __A) { - // CHECK-LABEL: @test_mm512_maskz_extractf32x8_ps + // CHECK-LABEL: test_mm512_maskz_extractf32x8_ps // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}} return _mm512_maskz_extractf32x8_ps(__U, __A, 1); } __m128d test_mm512_extractf64x2_pd(__m512d __A) { - // CHECK-LABEL: @test_mm512_extractf64x2_pd + // CHECK-LABEL: test_mm512_extractf64x2_pd // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> poison, <2 x i32> <i32 6, i32 7> return _mm512_extractf64x2_pd(__A, 3); } __m128d test_mm512_mask_extractf64x2_pd(__m128d __W, __mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_mask_extractf64x2_pd + // CHECK-LABEL: test_mm512_mask_extractf64x2_pd // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> poison, <2 x i32> <i32 6, i32 7> // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}} return _mm512_mask_extractf64x2_pd(__W, __U, __A, 3); } __m128d test_mm512_maskz_extractf64x2_pd(__mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_maskz_extractf64x2_pd + // CHECK-LABEL: test_mm512_maskz_extractf64x2_pd // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> poison, <2 x i32> <i32 6, i32 7> // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}} return _mm512_maskz_extractf64x2_pd(__U, __A, 3); } __m256i test_mm512_extracti32x8_epi32(__m512i __A) { - // CHECK-LABEL: @test_mm512_extracti32x8_epi32 + // CHECK-LABEL: test_mm512_extracti32x8_epi32 // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> return _mm512_extracti32x8_epi32(__A, 1); } __m256i test_mm512_mask_extracti32x8_epi32(__m256i __W, __mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_mask_extracti32x8_epi32 + // CHECK-LABEL: test_mm512_mask_extracti32x8_epi32 // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}} return _mm512_mask_extracti32x8_epi32(__W, __U, __A, 1); } __m256i test_mm512_maskz_extracti32x8_epi32(__mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_maskz_extracti32x8_epi32 + // CHECK-LABEL: test_mm512_maskz_extracti32x8_epi32 // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}} return _mm512_maskz_extracti32x8_epi32(__U, __A, 1); } __m128i test_mm512_extracti64x2_epi64(__m512i __A) { - // CHECK-LABEL: @test_mm512_extracti64x2_epi64 + // CHECK-LABEL: test_mm512_extracti64x2_epi64 // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> poison, <2 x i32> <i32 6, i32 7> return _mm512_extracti64x2_epi64(__A, 3); } __m128i test_mm512_mask_extracti64x2_epi64(__m128i __W, __mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_mask_extracti64x2_epi64 + // CHECK-LABEL: test_mm512_mask_extracti64x2_epi64 // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> poison, <2 x i32> <i32 6, i32 7> // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}} return _mm512_mask_extracti64x2_epi64(__W, __U, __A, 3); } __m128i test_mm512_maskz_extracti64x2_epi64(__mmask8 __U, __m512i __A) { - // CHECK-LABEL: @test_mm512_maskz_extracti64x2_epi64 + // CHECK-LABEL: test_mm512_maskz_extracti64x2_epi64 // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> poison, <2 x i32> <i32 6, i32 7> // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}} return _mm512_maskz_extracti64x2_epi64(__U, __A, 3); } __m512 test_mm512_insertf32x8(__m512 __A, __m256 __B) { - // CHECK-LABEL: @test_mm512_insertf32x8 + // CHECK-LABEL: test_mm512_insertf32x8 // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23> return _mm512_insertf32x8(__A, __B, 1); } __m512 test_mm512_mask_insertf32x8(__m512 __W, __mmask16 __U, __m512 __A, __m256 __B) { - // CHECK-LABEL: @test_mm512_mask_insertf32x8 + // CHECK-LABEL: test_mm512_mask_insertf32x8 // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23> // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}} return _mm512_mask_insertf32x8(__W, __U, __A, __B, 1); } __m512 test_mm512_maskz_insertf32x8(__mmask16 __U, __m512 __A, __m256 __B) { - // CHECK-LABEL: @test_mm512_maskz_insertf32x8 + // CHECK-LABEL: test_mm512_maskz_insertf32x8 // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23> // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}} return _mm512_maskz_insertf32x8(__U, __A, __B, 1); } __m512d test_mm512_insertf64x2(__m512d __A, __m128d __B) { - // CHECK-LABEL: @test_mm512_insertf64x2 + // CHECK-LABEL: test_mm512_insertf64x2 // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9> return _mm512_insertf64x2(__A, __B, 3); } __m512d test_mm512_mask_insertf64x2(__m512d __W, __mmask8 __U, __m512d __A, __m128d __B) { - // CHECK-LABEL: @test_mm512_mask_insertf64x2 + // CHECK-LABEL: test_mm512_mask_insertf64x2 // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9> // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} return _mm512_mask_insertf64x2(__W, __U, __A, __B, 3); } __m512d test_mm512_maskz_insertf64x2(__mmask8 __U, __m512d __A, __m128d __B) { - // CHECK-LABEL: @test_mm512_maskz_insertf64x2 + // CHECK-LABEL: test_mm512_maskz_insertf64x2 // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9> // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}} return _mm512_maskz_insertf64x2(__U, __A, __B, 3); } __m512i test_mm512_inserti32x8(__m512i __A, __m256i __B) { - // CHECK-LABEL: @test_mm512_inserti32x8 + // CHECK-LABEL: test_mm512_inserti32x8 // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23> return _mm512_inserti32x8(__A, __B, 1); } __m512i test_mm512_mask_inserti32x8(__m512i __W, __mmask16 __U, __m512i __A, __m256i __B) { - // CHECK-LABEL: @test_mm512_mask_inserti32x8 + // CHECK-LABEL: test_mm512_mask_inserti32x8 // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23> // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}} return _mm512_mask_inserti32x8(__W, __U, __A, __B, 1); } __m512i test_mm512_maskz_inserti32x8(__mmask16 __U, __m512i __A, __m256i __B) { - // CHECK-LABEL: @test_mm512_maskz_inserti32x8 + // CHECK-LABEL: test_mm512_maskz_inserti32x8 // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23> // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}} return _mm512_maskz_inserti32x8(__U, __A, __B, 1); } __m512i test_mm512_inserti64x2(__m512i __A, __m128i __B) { - // CHECK-LABEL: @test_mm512_inserti64x2 + // CHECK-LABEL: test_mm512_inserti64x2 // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7> return _mm512_inserti64x2(__A, __B, 1); } __m512i test_mm512_mask_inserti64x2(__m512i __W, __mmask8 __U, __m512i __A, __m128i __B) { - // CHECK-LABEL: @test_mm512_mask_inserti64x2 + // CHECK-LABEL: test_mm512_mask_inserti64x2 // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7> // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}} return _mm512_mask_inserti64x2(__W, __U, __A, __B, 1); } __m512i test_mm512_maskz_inserti64x2(__mmask8 __U, __m512i __A, __m128i __B) { - // CHECK-LABEL: @test_mm512_maskz_inserti64x2 + // CHECK-LABEL: test_mm512_maskz_inserti64x2 // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7> // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}} return _mm512_maskz_inserti64x2(__U, __A, __B, 1); } __mmask8 test_mm512_mask_fpclass_pd_mask(__mmask8 __U, __m512d __A) { - // CHECK-LABEL: @test_mm512_mask_fpclass_pd_mask + // CHECK-LABEL: test_mm512_mask_fpclass_pd_mask // CHECK: @llvm.x86.avx512.fpclass.pd.512 return _mm512_mask_fpclass_pd_mask(__U, __A, 4); } __mmask8 test_mm512_fpclass_pd_mask(__m512d __A) { - // CHECK-LABEL: @test_mm512_fpclass_pd_mask + // CHECK-LABEL: test_mm512_fpclass_pd_mask // CHECK: @llvm.x86.avx512.fpclass.pd.512 return _mm512_fpclass_pd_mask(__A, 4); } __mmask16 test_mm512_mask_fpclass_ps_mask(__mmask16 __U, __m512 __A) { - // CHECK-LABEL: @test_mm512_mask_fpclass_ps_mask + // CHECK-LABEL: test_mm512_mask_fpclass_ps_mask // CHECK: @llvm.x86.avx512.fpclass.ps.512 return _mm512_mask_fpclass_ps_mask(__U, __A, 4); } __mmask16 test_mm512_fpclass_ps_mask(__m512 __A) { - // CHECK-LABEL: @test_mm512_fpclass_ps_mask + // CHECK-LABEL: test_mm512_fpclass_ps_mask // CHECK: @llvm.x86.avx512.fpclass.ps.512 return _mm512_fpclass_ps_mask(__A, 4); } __mmask8 test_mm_fpclass_sd_mask(__m128d __A) { - // CHECK-LABEL: @test_mm_fpclass_sd_mask + // CHECK-LABEL: test_mm_fpclass_sd_mask // CHECK: @llvm.x86.avx512.mask.fpclass.sd return _mm_fpclass_sd_mask (__A, 2); } __mmask8 test_mm_mask_fpclass_sd_mask(__mmask8 __U, __m128d __A) { - // CHECK-LABEL: @test_mm_mask_fpclass_sd_mask + // CHECK-LABEL: test_mm_mask_fpclass_sd_mask // CHECK: @llvm.x86.avx512.mask.fpclass.sd return _mm_mask_fpclass_sd_mask (__U, __A, 2); } __mmask8 test_mm_fpclass_ss_mask(__m128 __A) { - // CHECK-LABEL: @test_mm_fpclass_ss_mask + // CHECK-LABEL: test_mm_fpclass_ss_mask // CHECK: @llvm.x86.avx512.mask.fpclass.ss return _mm_fpclass_ss_mask ( __A, 2); } __mmask8 test_mm_mask_fpclass_ss_mask(__mmask8 __U, __m128 __A) { - // CHECK-LABEL: @test_mm_mask_fpclass_ss_mask + // CHECK-LABEL: test_mm_mask_fpclass_ss_mask // CHECK: @llvm.x86.avx512.mask.fpclass.ss return _mm_mask_fpclass_ss_mask (__U, __A, 2); } diff --git a/clang/test/CodeGen/X86/avx512f-builtins.c b/clang/test/CodeGen/X86/avx512f-builtins.c index 84e700c..8c14c57 100644 --- a/clang/test/CodeGen/X86/avx512f-builtins.c +++ b/clang/test/CodeGen/X86/avx512f-builtins.c @@ -154,6 +154,7 @@ __m512 test_mm512_add_ps(__m512 a, __m512 b) // CHECK: fadd <16 x float> return _mm512_add_ps(a, b); } +TEST_CONSTEXPR(match_m512(_mm512_add_ps((__m512){-1.0f, -2.0f, -3.0f, -4.0f, -5.0f, -6.0f, -7.0f, -8.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f, +8.0f}, (__m512){-1.0f, -2.0f, -3.0f, -4.0f, -5.0f, -6.0f, -7.0f, -8.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f, +8.0f}), -2.0f, -4.0f, -6.0f, -8.0f, -10.0f, -12.0f, -14.0f, -16.0f, +2.0f, +4.0f, +6.0f, +8.0f, +10.0f, +12.0f, +14.0f, +16.0f)); __m512d test_mm512_add_pd(__m512d a, __m512d b) { @@ -161,6 +162,7 @@ __m512d test_mm512_add_pd(__m512d a, __m512d b) // CHECK: fadd <8 x double> return _mm512_add_pd(a, b); } +TEST_CONSTEXPR(match_m512d(_mm512_add_pd((__m512d){-1.0, -2.0, -3.0, -4.0, +1.0, +2.0, +3.0, +4.0}, (__m512d){-1.0, -2.0, -3.0, -4.0, +1.0, +2.0, +3.0, +4.0}), -2.0, -4.0, -6.0, -8.0, +2.0, +4.0, +6.0, +8.0)); __m512 test_mm512_mul_ps(__m512 a, __m512 b) { @@ -168,6 +170,7 @@ __m512 test_mm512_mul_ps(__m512 a, __m512 b) // CHECK: fmul <16 x float> return _mm512_mul_ps(a, b); } +TEST_CONSTEXPR(match_m512(_mm512_mul_ps((__m512){-1.0f, -2.0f, -3.0f, -4.0f, -5.0f, -6.0f, -7.0f, -8.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f, +8.0f}, (__m512){-1.0f, -2.0f, -3.0f, -4.0f, -5.0f, -6.0f, -7.0f, -8.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f, +8.0f}), +1.0f, +4.0f, +9.0f, +16.0f, +25.0f, +36.0f, +49.0f, +64.0f, +1.0f, +4.0f, +9.0f, +16.0f, +25.0f, +36.0f, +49.0f, +64.0f)); __m512d test_mm512_mul_pd(__m512d a, __m512d b) { @@ -175,6 +178,7 @@ __m512d test_mm512_mul_pd(__m512d a, __m512d b) // CHECK: fmul <8 x double> return _mm512_mul_pd(a, b); } +TEST_CONSTEXPR(match_m512d(_mm512_mul_pd((__m512d){-1.0, -2.0, -3.0, -4.0, +1.0, +2.0, +3.0, +4.0}, (__m512d){-1.0, -2.0, -3.0, -4.0, +1.0, +2.0, +3.0, +4.0}), +1.0, +4.0, +9.0, +16.0, +1.0, +4.0, +9.0, +16.0)); void test_mm512_storeu_si512 (void *__P, __m512i __A) { @@ -1261,6 +1265,7 @@ __m512d test_mm512_unpackhi_pd(__m512d a, __m512d b) // CHECK: shufflevector <8 x double> {{.*}} <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15> return _mm512_unpackhi_pd(a, b); } +TEST_CONSTEXPR(match_m512d(_mm512_unpackhi_pd((__m512d){1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}, (__m512d){9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0}), +2.0, +10.0, +4.0, +12.0, +6.0, +14.0, +8.0, +16.0)); __m512d test_mm512_unpacklo_pd(__m512d a, __m512d b) { @@ -1268,6 +1273,7 @@ __m512d test_mm512_unpacklo_pd(__m512d a, __m512d b) // CHECK: shufflevector <8 x double> {{.*}} <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14> return _mm512_unpacklo_pd(a, b); } +TEST_CONSTEXPR(match_m512d(_mm512_unpacklo_pd((__m512d){1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}, (__m512d){9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0}), +1.0, +9.0, +3.0, +11.0, +5.0, +13.0, +7.0, +15.0)); __m512 test_mm512_unpackhi_ps(__m512 a, __m512 b) { @@ -1275,6 +1281,7 @@ __m512 test_mm512_unpackhi_ps(__m512 a, __m512 b) // CHECK: shufflevector <16 x float> {{.*}} <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31> return _mm512_unpackhi_ps(a, b); } +TEST_CONSTEXPR(match_m512(_mm512_unpackhi_ps((__m512){0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f}, (__m512){16.0f, 17.0f, 18.0f, 19.0f, 20.0f, 21.0f, 22.0f, 23.0f, 24.0f, 25.0f, 26.0f, 27.0f, 28.0f, 29.0f, 30.0f, 31.0f}), +2.0f, +18.0f, +3.0f, +19.0f, +6.0f, +22.0f, +7.0f, +23.0f, +10.0f, +26.0f, +11.0f, +27.0f, +14.0f, +30.0f, +15.0f, +31.0f)); __m512 test_mm512_unpacklo_ps(__m512 a, __m512 b) { @@ -1282,6 +1289,7 @@ __m512 test_mm512_unpacklo_ps(__m512 a, __m512 b) // CHECK: shufflevector <16 x float> {{.*}} <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29> return _mm512_unpacklo_ps(a, b); } +TEST_CONSTEXPR(match_m512(_mm512_unpacklo_ps((__m512){0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f}, (__m512){16.0f, 17.0f, 18.0f, 19.0f, 20.0f, 21.0f, 22.0f, 23.0f, 24.0f, 25.0f, 26.0f, 27.0f, 28.0f, 29.0f, 30.0f, 31.0f}), +0.0f, +16.0f, +1.0f, +17.0f, +4.0f, +20.0f, +5.0f, +21.0f, +8.0f, +24.0f, +9.0f, +25.0f, +12.0f, +28.0f, +13.0f, +29.0f)); __mmask16 test_mm512_cmp_round_ps_mask(__m512 a, __m512 b) { // CHECK-LABEL: test_mm512_cmp_round_ps_mask @@ -2811,36 +2819,42 @@ __m512i test_mm512_and_epi32(__m512i __src,__mmask16 __k, __m512i __a, __m512i _ // CHECK: and <16 x i32> return _mm512_and_epi32(__a, __b); } +TEST_CONSTEXPR(match_v16si(_mm512_and_epi32((__m512i)(__v16si){0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v16si){0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1}), 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0, -1)); __m512i test_mm512_and_epi64(__m512i __src,__mmask8 __k, __m512i __a, __m512i __b) { // CHECK-LABEL: test_mm512_and_epi64 // CHECK: and <8 x i64> return _mm512_and_epi64(__a, __b); } +TEST_CONSTEXPR(match_v8di(_mm512_and_epi64((__m512i)(__v8di){0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v8di){0, 0, -1, -1, 0, -1, 0, -1}), 0, 0, 0, -1, 0, 0, 0, -1)); __m512i test_mm512_or_epi32(__m512i __src,__mmask16 __k, __m512i __a, __m512i __b) { // CHECK-LABEL: test_mm512_or_epi32 // CHECK: or <16 x i32> return _mm512_or_epi32(__a, __b); } +TEST_CONSTEXPR(match_v16si(_mm512_or_epi32((__m512i)(__v16si){0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v16si){0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1}), 0, -1, -1, -1, 0, -1, -1, -1, 0, -1, -1, -1, 0, -1, -1, -1)); __m512i test_mm512_or_epi64(__m512i __src,__mmask8 __k, __m512i __a, __m512i __b) { // CHECK-LABEL: test_mm512_or_epi64 // CHECK: or <8 x i64> return _mm512_or_epi64(__a, __b); } +TEST_CONSTEXPR(match_v8di(_mm512_or_epi64((__m512i)(__v8di){0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v8di){0, 0, -1, -1, 0, -1, 0, -1}), 0, -1, -1, -1, 0, -1, -1, -1)); __m512i test_mm512_xor_epi32(__m512i __src,__mmask16 __k, __m512i __a, __m512i __b) { // CHECK-LABEL: test_mm512_xor_epi32 // CHECK: xor <16 x i32> return _mm512_xor_epi32(__a, __b); } +TEST_CONSTEXPR(match_v16si(_mm512_xor_epi32((__m512i)(__v16si){0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v16si){0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1}), 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0)); __m512i test_mm512_xor_epi64(__m512i __src,__mmask8 __k, __m512i __a, __m512i __b) { // CHECK-LABEL: test_mm512_xor_epi64 // CHECK: xor <8 x i64> return _mm512_xor_epi64(__a, __b); } +TEST_CONSTEXPR(match_v8di(_mm512_xor_epi64((__m512i)(__v8di){0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v8di){0, 0, -1, -1, 0, -1, 0, -1}), 0, -1, -1, 0, 0, -1, -1, 0)); __m512i test_mm512_maskz_andnot_epi32 (__mmask16 __k,__m512i __A, __m512i __B){ // CHECK-LABEL: test_mm512_maskz_andnot_epi32 @@ -2869,6 +2883,7 @@ __m512i test_mm512_andnot_si512(__m512i __A, __m512i __B) return _mm512_andnot_si512(__A, __B); } +TEST_CONSTEXPR(match_v8di(_mm512_andnot_si512((__m512i)(__v8di){0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v8di){0, 0, -1, -1, 0, -1, 0, -1}), 0, 0, -1, 0, 0, -1, 0, 0)); __m512i test_mm512_andnot_epi32(__m512i __A, __m512i __B) { // CHECK-LABEL: test_mm512_andnot_epi32 @@ -2876,6 +2891,7 @@ __m512i test_mm512_andnot_epi32(__m512i __A, __m512i __B) { // CHECK: and <16 x i32> %{{.*}}, %{{.*}} return _mm512_andnot_epi32(__A,__B); } +TEST_CONSTEXPR(match_v16si(_mm512_andnot_epi32((__m512i)(__v16si){0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v16si){0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1}), 0, 0, -1, 0, 0, -1, 0, 0, 0, 0, -1, 0, 0, -1, 0, 0)); __m512i test_mm512_maskz_andnot_epi64 (__mmask8 __k,__m512i __A, __m512i __B) { // CHECK-LABEL: test_mm512_maskz_andnot_epi64 @@ -2900,6 +2916,7 @@ __m512i test_mm512_andnot_epi64(__m512i __A, __m512i __B) { // CHECK: and <8 x i64> %{{.*}}, %{{.*}} return _mm512_andnot_epi64(__A,__B); } +TEST_CONSTEXPR(match_v8di(_mm512_andnot_epi64((__m512i)(__v8di){0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v8di){0, 0, -1, -1, 0, -1, 0, -1}), 0, 0, -1, 0, 0, -1, 0, 0)); __m512i test_mm512_maskz_sub_epi32 (__mmask16 __k,__m512i __A, __m512i __B) { //CHECK-LABEL: test_mm512_maskz_sub_epi32 @@ -3542,6 +3559,7 @@ __m512d test_mm512_div_pd(__m512d __a, __m512d __b) { // CHECK: fdiv <8 x double> return _mm512_div_pd(__a,__b); } +TEST_CONSTEXPR(match_m512d(_mm512_div_pd((__m512d){+8.0, +6.0, +4.0, +2.0, -8.0, -6.0, -4.0, -2.0}, (__m512d){+2.0, +2.0, +2.0, +2.0, -2.0, -2.0, -2.0, -2.0}), +4.0, +3.0, +2.0, +1.0, +4.0, +3.0, +2.0, +1.0)); __m512d test_mm512_mask_div_pd(__m512d __w, __mmask8 __u, __m512d __a, __m512d __b) { // CHECK-LABEL: test_mm512_mask_div_pd // CHECK: fdiv <8 x double> %{{.*}}, %{{.*}} @@ -3576,6 +3594,7 @@ __m512 test_mm512_div_ps(__m512 __A, __m512 __B) { // CHECK: fdiv <16 x float> return _mm512_div_ps(__A,__B); } +TEST_CONSTEXPR(match_m512(_mm512_div_ps((__m512){+16.0f, +14.0f, +12.0f, +10.0f, +8.0f, +6.0f, +4.0f, +2.0f, -16.0f, -14.0f, -12.0f, -10.0f, -8.0f, -6.0f, -4.0f, -2.0f}, (__m512){+2.0f, +2.0f, +2.0f, +2.0f, +2.0f, +2.0f, +2.0f, +2.0f, -2.0f, -2.0f, -2.0f, -2.0f, -2.0f, -2.0f, -2.0f, -2.0f}), +8.0f, +7.0f, +6.0f, +5.0f, +4.0f, +3.0f, +2.0f, +1.0f, +8.0f, +7.0f, +6.0f, +5.0f, +4.0f, +3.0f, +2.0f, +1.0f)); __m512 test_mm512_mask_div_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { // CHECK-LABEL: test_mm512_mask_div_ps // CHECK: fdiv <16 x float> %{{.*}}, %{{.*}} @@ -4386,6 +4405,7 @@ __m512d test_mm512_movedup_pd(__m512d __A) { // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> %{{.*}}, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6> return _mm512_movedup_pd(__A); } +TEST_CONSTEXPR(match_m512d(_mm512_movedup_pd((__m512d){-1.0, +2.0, +3.0, +4.0, -5.0, -6.0, +7.0, +8.0}), -1.0, -1.0, +3.0, +3.0, -5.0, -5.0, +7.0, +7.0)); __m512d test_mm512_mask_movedup_pd(__m512d __W, __mmask8 __U, __m512d __A) { // CHECK-LABEL: test_mm512_mask_movedup_pd @@ -8682,6 +8702,7 @@ __m512 test_mm512_movehdup_ps(__m512 __A) { // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15> return _mm512_movehdup_ps(__A); } +TEST_CONSTEXPR(match_m512(_mm512_movehdup_ps((__m512){+1.0f,-1.0f,+2.0f,-2.0f,+3.0f,-3.0f,+4.0f,-4.0f,+5.0f,-5.0f,+6.0f,-6.0f,+7.0f,-7.0f,+8.0f,-8.0f}), -1.0f, -1.0f, -2.0f, -2.0f, -3.0f, -3.0f, -4.0f, -4.0f, -5.0f, -5.0f, -6.0f, -6.0f, -7.0f, -7.0f, -8.0f, -8.0f)); __m512 test_mm512_mask_movehdup_ps(__m512 __W, __mmask16 __U, __m512 __A) { // CHECK-LABEL: test_mm512_mask_movehdup_ps @@ -8702,6 +8723,7 @@ __m512 test_mm512_moveldup_ps(__m512 __A) { // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14> return _mm512_moveldup_ps(__A); } +TEST_CONSTEXPR(match_m512(_mm512_moveldup_ps((__m512){+1.0f,-1.0f,+2.0f,-2.0f,+3.0f,-3.0f,+4.0f,-4.0f,+5.0f,-5.0f,+6.0f,-6.0f,+7.0f,-7.0f,+8.0f,-8.0f}), +1.0f, +1.0f, +2.0f, +2.0f, +3.0f, +3.0f, +4.0f, +4.0f, +5.0f, +5.0f, +6.0f, +6.0f, +7.0f, +7.0f, +8.0f, +8.0f)); __m512 test_mm512_mask_moveldup_ps(__m512 __W, __mmask16 __U, __m512 __A) { // CHECK-LABEL: test_mm512_mask_moveldup_ps diff --git a/clang/test/CodeGen/X86/mmx-builtins.c b/clang/test/CodeGen/X86/mmx-builtins.c index 52cbe45..30e2cfe 100644 --- a/clang/test/CodeGen/X86/mmx-builtins.c +++ b/clang/test/CodeGen/X86/mmx-builtins.c @@ -34,24 +34,28 @@ __m64 test_mm_add_pi8(__m64 a, __m64 b) { // CHECK: add <8 x i8> {{%.*}}, {{%.*}} return _mm_add_pi8(a, b); } +TEST_CONSTEXPR(match_v8qi(_mm_add_pi8(_mm_setr_pi8(-3, +2, -1, 0, +1, -2, +3, -4), _mm_setr_pi8(-18, +16, -14, +12, -10, +8, +6, -4)), -21, +18, -15, +12, -9, +6, +9, -8)); __m64 test_mm_add_pi16(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_add_pi16 // CHECK: add <4 x i16> {{%.*}}, {{%.*}} return _mm_add_pi16(a, b); } +TEST_CONSTEXPR(match_v4hi(_mm_add_pi16((__m64)(__v4hi){+1, -2, +3, -4}, (__m64)(__v4hi){-10, +8, +6, -4}), -9, +6, +9, -8)); __m64 test_mm_add_pi32(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_add_pi32 // CHECK: add <2 x i32> {{%.*}}, {{%.*}} return _mm_add_pi32(a, b); } +TEST_CONSTEXPR(match_v2si(_mm_add_pi32((__m64)(__v2si){+5, -3}, (__m64)(__v2si){-9, +8}), -4, +5)); __m64 test_mm_add_si64(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_add_si64 // CHECK: add i64 {{%.*}}, {{%.*}} return _mm_add_si64(a, b); } +TEST_CONSTEXPR(match_v1di(_mm_add_si64((__m64)(__v1di){+42}, (__m64)(__v1di){-100}), -58)); __m64 test_mm_adds_pi8(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_adds_pi8 @@ -88,6 +92,7 @@ __m64 test_mm_and_si64(__m64 a, __m64 b) { // CHECK: and <1 x i64> {{%.*}}, {{%.*}} return _mm_and_si64(a, b); } +TEST_CONSTEXPR(match_v4hi(_mm_and_si64((__m64)(__v4hi){0, -1, 0, -1}, (__m64)(__v4hi){0, 0, -1, -1}), 0, 0, 0, -1)); __m64 test_mm_andnot_si64(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_andnot_si64 @@ -95,6 +100,7 @@ __m64 test_mm_andnot_si64(__m64 a, __m64 b) { // CHECK: and <1 x i64> [[TMP]], {{%.*}} return _mm_andnot_si64(a, b); } +TEST_CONSTEXPR(match_v4hi(_mm_andnot_si64((__m64)(__v4hi){0, -1, 0, -1}, (__m64)(__v4hi){0, 0, -1, -1}), 0, 0, -1, 0)); __m64 test_mm_avg_pu8(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_avg_pu8 @@ -114,6 +120,7 @@ __m64 test_mm_cmpeq_pi8(__m64 a, __m64 b) { // CHECK-NEXT: {{%.*}} = sext <8 x i1> [[CMP]] to <8 x i8> return _mm_cmpeq_pi8(a, b); } +TEST_CONSTEXPR(match_v8qi(_mm_cmpeq_pi8(_mm_setr_pi8(-3, +2, -1, 0, +1, -2, +3, -4), _mm_setr_pi8(-3, -2, +1, 0, -1, -2, -3, -4)), -1, 0, 0, -1, 0, -1, 0, -1)); __m64 test_mm_cmpeq_pi16(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_cmpeq_pi16 @@ -121,6 +128,7 @@ __m64 test_mm_cmpeq_pi16(__m64 a, __m64 b) { // CHECK-NEXT: {{%.*}} = sext <4 x i1> [[CMP]] to <4 x i16> return _mm_cmpeq_pi16(a, b); } +TEST_CONSTEXPR(match_v4hi(_mm_cmpeq_pi16((__m64)(__v4hi){+1, -2, +3, -4}, (__m64)(__v4hi){-1, -1, +3, +4}), 0, 0, -1, 0)); __m64 test_mm_cmpeq_pi32(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_cmpeq_pi32 @@ -128,6 +136,7 @@ __m64 test_mm_cmpeq_pi32(__m64 a, __m64 b) { // CHECK-NEXT: {{%.*}} = sext <2 x i1> [[CMP]] to <2 x i32> return _mm_cmpeq_pi32(a, b); } +TEST_CONSTEXPR(match_v2si(_mm_cmpeq_pi32((__m64)(__v2si){+5, -3}, (__m64)(__v2si){-5, -3}), 0, -1)); __m64 test_mm_cmpgt_pi8(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_cmpgt_pi8 @@ -135,6 +144,7 @@ __m64 test_mm_cmpgt_pi8(__m64 a, __m64 b) { // CHECK-NEXT: {{%.*}} = sext <8 x i1> [[CMP]] to <8 x i8> return _mm_cmpgt_pi8(a, b); } +TEST_CONSTEXPR(match_v8qi(_mm_cmpgt_pi8(_mm_setr_pi8(-3, +2, -1, 0, +1, -2, +3, -4), _mm_setr_pi8(-3, -2, +1, 0, -1, -2, -3, -4)), 0, -1, 0, 0, -1, 0, -1, 0)); __m64 test_mm_cmpgt_pi16(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_cmpgt_pi16 @@ -142,6 +152,7 @@ __m64 test_mm_cmpgt_pi16(__m64 a, __m64 b) { // CHECK-NEXT: {{%.*}} = sext <4 x i1> [[CMP]] to <4 x i16> return _mm_cmpgt_pi16(a, b); } +TEST_CONSTEXPR(match_v4hi(_mm_cmpgt_pi16((__m64)(__v4hi){+1, -2, +3, -4}, (__m64)(__v4hi){-1, -1, +3, +4}), -1, 0, 0, 0)); __m64 test_mm_cmpgt_pi32(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_cmpgt_pi32 @@ -149,6 +160,7 @@ __m64 test_mm_cmpgt_pi32(__m64 a, __m64 b) { // CHECK-NEXT: {{%.*}} = sext <2 x i1> [[CMP]] to <2 x i32> return _mm_cmpgt_pi32(a, b); } +TEST_CONSTEXPR(match_v2si(_mm_cmpgt_pi32((__m64)(__v2si){+5, -3}, (__m64)(__v2si){-5, -3}), -1, 0)); __m128 test_mm_cvt_pi2ps(__m128 a, __m64 b) { // CHECK-LABEL: test_mm_cvt_pi2ps @@ -210,12 +222,14 @@ __m64 test_mm_cvtsi32_si64(int a) { // CHECK: insertelement <2 x i32> return _mm_cvtsi32_si64(a); } +TEST_CONSTEXPR(match_v2si(_mm_cvtsi32_si64(-127), -127, 0)); int test_mm_cvtsi64_si32(__m64 a) { // CHECK-LABEL: test_mm_cvtsi64_si32 // CHECK: extractelement <2 x i32> return _mm_cvtsi64_si32(a); } +TEST_CONSTEXPR(_mm_cvtsi64_si32((__m64)(__v4hi){-2, 0, -1, -1}) == 65534); __m64 test_mm_cvttpd_pi32(__m128d a) { // CHECK-LABEL: test_mm_cvttpd_pi32 @@ -240,11 +254,13 @@ __m64 test_m_from_int(int a) { // CHECK: insertelement <2 x i32> return _m_from_int(a); } +TEST_CONSTEXPR(match_v2si(_m_from_int(255), 255, 0)); __m64 test_m_from_int64(long long a) { // CHECK-LABEL: test_m_from_int64 return _m_from_int64(a); } +TEST_CONSTEXPR(match_v1di(_m_from_int64(-65536), -65536LL)); __m64 test_mm_hadd_pi16(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_hadd_pi16 @@ -367,12 +383,14 @@ __m64 test_mm_mullo_pi16(__m64 a, __m64 b) { // CHECK: mul <4 x i16> {{%.*}}, {{%.*}} return _mm_mullo_pi16(a, b); } +TEST_CONSTEXPR(match_v4hi(_mm_mullo_pi16((__m64)(__v4hi){+1, -2, +3, -4}, (__m64)(__v4hi){-10, +8, +6, -4}), -10, -16, +18, +16)); __m64 test_mm_or_si64(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_or_si64 // CHECK: or <1 x i64> {{%.*}}, {{%.*}} return _mm_or_si64(a, b); } +TEST_CONSTEXPR(match_v4hi(_mm_or_si64((__m64)(__v4hi){0, -1, 0, -1}, (__m64)(__v4hi){0, 0, -1, -1}), 0, -1, -1, -1)); __m64 test_mm_packs_pi16(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_packs_pi16 @@ -644,24 +662,28 @@ __m64 test_mm_sub_pi8(__m64 a, __m64 b) { // CHECK: sub <8 x i8> {{%.*}}, {{%.*}} return _mm_sub_pi8(a, b); } +TEST_CONSTEXPR(match_v8qi(_mm_sub_pi8(_mm_setr_pi8(-3, +2, -1, 0, +1, -2, +3, -4), _mm_setr_pi8(-18, +16, -14, +12, -10, +8, +6, -4)), +15, -14, +13, -12, +11, -10, -3, 0)); __m64 test_mm_sub_pi16(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_sub_pi16 // CHECK: sub <4 x i16> {{%.*}}, {{%.*}} return _mm_sub_pi16(a, b); } +TEST_CONSTEXPR(match_v4hi(_mm_sub_pi16((__m64)(__v4hi){+1, -2, +3, -4}, (__m64)(__v4hi){-10, +8, +6, -4}), +11, -10, -3, 0)); __m64 test_mm_sub_pi32(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_sub_pi32 // CHECK: sub <2 x i32> {{%.*}}, {{%.*}} return _mm_sub_pi32(a, b); } +TEST_CONSTEXPR(match_v2si(_mm_sub_pi32((__m64)(__v2si){+5, -3}, (__m64)(__v2si){-9, +8}), +14, -11)); __m64 test_mm_sub_si64(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_sub_si64 // CHECK: sub i64 {{%.*}}, {{%.*}} return _mm_sub_si64(a, b); } +TEST_CONSTEXPR(match_v1di(_mm_sub_si64((__m64)(__v1di){+42}, (__m64)(__v1di){-100}), +142)); __m64 test_mm_subs_pi8(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_subs_pi8 @@ -692,11 +714,13 @@ int test_m_to_int(__m64 a) { // CHECK: extractelement <2 x i32> return _m_to_int(a); } +TEST_CONSTEXPR(_m_to_int((__m64)(__v4hi){0, -2, -1, -1}) == -131072); long long test_m_to_int64(__m64 a) { // CHECK-LABEL: test_m_to_int64 return _m_to_int64(a); } +TEST_CONSTEXPR(_m_to_int64((__m64)(__v4hi){0, -2, 0, -1}) == -281470681874432LL); __m64 test_mm_unpackhi_pi8(__m64 a, __m64 b) { // CHECK-LABEL: test_mm_unpackhi_pi8 @@ -739,3 +763,4 @@ __m64 test_mm_xor_si64(__m64 a, __m64 b) { // CHECK: xor <1 x i64> {{%.*}}, {{%.*}} return _mm_xor_si64(a, b); } +TEST_CONSTEXPR(match_v4hi(_mm_xor_si64((__m64)(__v4hi){0, -1, 0, -1}, (__m64)(__v4hi){0, 0, -1, -1}), 0, -1, -1, 0)); diff --git a/clang/test/CodeGen/X86/sse2-builtins.c b/clang/test/CodeGen/X86/sse2-builtins.c index 04df59e..affb6e7 100644 --- a/clang/test/CodeGen/X86/sse2-builtins.c +++ b/clang/test/CodeGen/X86/sse2-builtins.c @@ -96,6 +96,7 @@ __m128i test_mm_and_si128(__m128i A, __m128i B) { // CHECK: and <2 x i64> return _mm_and_si128(A, B); } +TEST_CONSTEXPR(match_v4si(_mm_and_si128((__m128i)(__v4si){0, -1, 0, -1}, (__m128i)(__v4si){0, 0, -1, -1}), 0, 0, 0, -1)); __m128d test_mm_andnot_pd(__m128d A, __m128d B) { // CHECK-LABEL: test_mm_andnot_pd @@ -111,6 +112,7 @@ __m128i test_mm_andnot_si128(__m128i A, __m128i B) { // CHECK: and <2 x i64> return _mm_andnot_si128(A, B); } +TEST_CONSTEXPR(match_v4si(_mm_andnot_si128((__m128i)(__v4si){0, -1, 0, -1}, (__m128i)(__v4si){0, 0, -1, -1}), 0, 0, -1, 0)); __m128i test_mm_avg_epu8(__m128i A, __m128i B) { // CHECK-LABEL: test_mm_avg_epu8 @@ -963,6 +965,7 @@ __m128i test_mm_or_si128(__m128i A, __m128i B) { // CHECK: or <2 x i64> %{{.*}}, %{{.*}} return _mm_or_si128(A, B); } +TEST_CONSTEXPR(match_v4si(_mm_or_si128((__m128i)(__v4si){0, -1, 0, -1}, (__m128i)(__v4si){0, 0, -1, -1}), 0, -1, -1, -1)); __m128i test_mm_packs_epi16(__m128i A, __m128i B) { // CHECK-LABEL: test_mm_packs_epi16 @@ -1831,3 +1834,4 @@ __m128i test_mm_xor_si128(__m128i A, __m128i B) { // CHECK: xor <2 x i64> %{{.*}}, %{{.*}} return _mm_xor_si128(A, B); } +TEST_CONSTEXPR(match_v4si(_mm_xor_si128((__m128i)(__v4si){0, -1, 0, -1}, (__m128i)(__v4si){0, 0, -1, -1}), 0, -1, -1, 0)); diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt index 332c087..dc2db1d 100644 --- a/flang-rt/lib/runtime/CMakeLists.txt +++ b/flang-rt/lib/runtime/CMakeLists.txt @@ -251,19 +251,33 @@ else() add_win_flangrt_runtime(STATIC dynamic MultiThreadedDLL INSTALL_WITH_TOOLCHAIN) add_win_flangrt_runtime(STATIC dynamic_dbg MultiThreadedDebugDLL INSTALL_WITH_TOOLCHAIN) - # Unittests link against LLVMSupport which is using CMake's default runtime - # library selection, which is either MultiThreadedDLL or MultiThreadedDebugDLL - # depending on the configuration. They have to match or linking will fail. + # Unittests link against LLVMSupport. If CMAKE_MSVC_RUNTIME_LIBRARY is set, + # that will have been used for LLVMSupport so it must also be used here. + # Otherwise this will use CMake's default runtime library selection, which + # is either MultiThreadedDLL or MultiThreadedDebugDLL depending on the configuration. + # They have to match or linking will fail. if (GENERATOR_IS_MULTI_CONFIG) # We cannot select an ALIAS library because it may be different # per configuration. Fallback to CMake's default. add_win_flangrt_runtime(STATIC unittest "" EXCLUDE_FROM_ALL) else () - string(TOLOWER ${CMAKE_BUILD_TYPE} build_type) - if (build_type STREQUAL "debug") - add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.dynamic_dbg) - else () - add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.dynamic) - endif () + # Check if CMAKE_MSVC_RUNTIME_LIBRARY was set. + if (CMAKE_MSVC_RUNTIME_LIBRARY STREQUAL "MultiThreaded") + add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.static) + elseif (CMAKE_MSVC_RUNTIME_LIBRARY STREQUAL "MultiThreadedDLL") + add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.dynamic) + elseif (CMAKE_MSVC_RUNTIME_LIBRARY STREQUAL "MultiThreadedDebug") + add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.static_dbg) + elseif (CMAKE_MSVC_RUNTIME_LIBRARY STREQUAL "MultiThreadedDebugDLL") + add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.dynamic_dbg) + else() + # Default based on the build type. + string(TOLOWER ${CMAKE_BUILD_TYPE} build_type) + if (build_type STREQUAL "debug") + add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.dynamic_dbg) + else () + add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.dynamic) + endif () + endif() endif () endif() diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt index 831bc8a..fd63ad1 100644 --- a/flang-rt/unittests/CMakeLists.txt +++ b/flang-rt/unittests/CMakeLists.txt @@ -94,14 +94,6 @@ function(add_flangrt_unittest test_dirname) target_link_libraries(${test_dirname} PRIVATE ${ARG_LINK_LIBS}) add_flangrt_unittest_offload_properties(${test_dirname}) add_flangrt_dependent_libs(${test_dirname}) - - # Required because LLVMSupport is compiled with this option. - # FIXME: According to CMake documentation, this is the default. Why is it - # needed? LLVM's add_unittest doesn't set it either. - set_target_properties(${test_dirname} - PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL" - ) endfunction() function(add_flangrt_nongtest_unittest test_name) diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt index adfabf4..756e097 100644 --- a/libclc/CMakeLists.txt +++ b/libclc/CMakeLists.txt @@ -31,6 +31,7 @@ set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS clc/lib/amdgcn/SOURCES; clc/lib/amdgpu/SOURCES; clc/lib/clspv/SOURCES; + clc/lib/ptx-nvidiacl/SOURCES; clc/lib/r600/SOURCES; clc/lib/spirv/SOURCES; ) diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index 51444ec..c6b87a3 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -447,14 +447,12 @@ set(files __fwd/get.h __fwd/ios.h __fwd/istream.h - __fwd/map.h __fwd/mdspan.h __fwd/memory.h __fwd/memory_resource.h __fwd/ostream.h __fwd/pair.h __fwd/queue.h - __fwd/set.h __fwd/span.h __fwd/sstream.h __fwd/stack.h diff --git a/libcxx/include/__fwd/map.h b/libcxx/include/__fwd/map.h deleted file mode 100644 index 940298d..0000000 --- a/libcxx/include/__fwd/map.h +++ /dev/null @@ -1,31 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef _LIBCPP___FWD_MAP_H -#define _LIBCPP___FWD_MAP_H - -#include <__config> -#include <__fwd/functional.h> -#include <__fwd/memory.h> -#include <__fwd/pair.h> - -#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) -# pragma GCC system_header -#endif - -_LIBCPP_BEGIN_NAMESPACE_STD - -template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > > -class map; - -template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > > -class multimap; - -_LIBCPP_END_NAMESPACE_STD - -#endif // _LIBCPP___FWD_MAP_H diff --git a/libcxx/include/__fwd/set.h b/libcxx/include/__fwd/set.h deleted file mode 100644 index d5ef8d5..0000000 --- a/libcxx/include/__fwd/set.h +++ /dev/null @@ -1,30 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef _LIBCPP___FWD_SET_H -#define _LIBCPP___FWD_SET_H - -#include <__config> -#include <__fwd/functional.h> -#include <__fwd/memory.h> - -#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) -# pragma GCC system_header -#endif - -_LIBCPP_BEGIN_NAMESPACE_STD - -template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> > -class set; - -template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> > -class multiset; - -_LIBCPP_END_NAMESPACE_STD - -#endif // _LIBCPP___FWD_SET_H diff --git a/libcxx/include/__tree b/libcxx/include/__tree index 74b20d5..3dd5ae5 100644 --- a/libcxx/include/__tree +++ b/libcxx/include/__tree @@ -13,9 +13,7 @@ #include <__algorithm/min.h> #include <__assert> #include <__config> -#include <__fwd/map.h> #include <__fwd/pair.h> -#include <__fwd/set.h> #include <__iterator/distance.h> #include <__iterator/iterator_traits.h> #include <__iterator/next.h> @@ -686,16 +684,6 @@ private: friend class __tree; template <class, class, class> friend class __tree_const_iterator; - template <class> - friend class __map_iterator; - template <class, class, class, class> - friend class map; - template <class, class, class, class> - friend class multimap; - template <class, class, class> - friend class set; - template <class, class, class> - friend class multiset; }; template <class _Tp, class _NodePtr, class _DiffType> @@ -709,18 +697,15 @@ class __tree_const_iterator { __end_node_pointer __ptr_; public: - using iterator_category = bidirectional_iterator_tag; - using value_type = __get_node_value_type_t<_Tp>; - using difference_type = _DiffType; - using reference = const value_type&; - using pointer = __rebind_pointer_t<_NodePtr, const value_type>; + using iterator_category = bidirectional_iterator_tag; + using value_type = __get_node_value_type_t<_Tp>; + using difference_type = _DiffType; + using reference = const value_type&; + using pointer = __rebind_pointer_t<_NodePtr, const value_type>; + using __non_const_iterator _LIBCPP_NODEBUG = __tree_iterator<_Tp, __node_pointer, difference_type>; _LIBCPP_HIDE_FROM_ABI __tree_const_iterator() _NOEXCEPT : __ptr_(nullptr) {} -private: - typedef __tree_iterator<_Tp, __node_pointer, difference_type> __non_const_iterator; - -public: _LIBCPP_HIDE_FROM_ABI __tree_const_iterator(__non_const_iterator __p) _NOEXCEPT : __ptr_(__p.__ptr_) {} _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_np()->__value_; } @@ -762,16 +747,6 @@ private: template <class, class, class> friend class __tree; - template <class, class, class, class> - friend class map; - template <class, class, class, class> - friend class multimap; - template <class, class, class> - friend class set; - template <class, class, class> - friend class multiset; - template <class> - friend class __map_const_iterator; }; template <class _Tp, class _Compare> diff --git a/libcxx/include/map b/libcxx/include/map index 0a43bd0..6378218 100644 --- a/libcxx/include/map +++ b/libcxx/include/map @@ -582,7 +582,6 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred); // C++20 # include <__functional/binary_function.h> # include <__functional/is_transparent.h> # include <__functional/operations.h> -# include <__fwd/map.h> # include <__iterator/erase_if_container.h> # include <__iterator/iterator_traits.h> # include <__iterator/ranges_iterator_traits.h> @@ -861,7 +860,10 @@ public: friend class __tree_const_iterator; }; -template <class _Key, class _Tp, class _Compare, class _Allocator> +template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > > +class multimap; + +template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > > class map { public: // types: diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in index b07a153e..c431c0c 100644 --- a/libcxx/include/module.modulemap.in +++ b/libcxx/include/module.modulemap.in @@ -1607,7 +1607,6 @@ module std [system] { } module map { - module fwd { header "__fwd/map.h" } header "map" export * export std.iterator.reverse_iterator @@ -1974,7 +1973,6 @@ module std [system] { } module set { - module fwd { header "__fwd/set.h" } header "set" export * export std.iterator.reverse_iterator diff --git a/libcxx/include/set b/libcxx/include/set index 342a529..c77345b 100644 --- a/libcxx/include/set +++ b/libcxx/include/set @@ -522,7 +522,6 @@ erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred); // C++20 # include <__config> # include <__functional/is_transparent.h> # include <__functional/operations.h> -# include <__fwd/set.h> # include <__iterator/erase_if_container.h> # include <__iterator/iterator_traits.h> # include <__iterator/ranges_iterator_traits.h> @@ -570,7 +569,10 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -template <class _Key, class _Compare, class _Allocator> +template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> > +class multiset; + +template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> > class set { public: // types: diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt index 4c70b98..e9a6faa 100644 --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -187,28 +187,29 @@ if ("lldb" IN_LIST LLVM_ENABLE_PROJECTS) endif () if ("libc" IN_LIST LLVM_ENABLE_PROJECTS) - message(WARNING "Using LLVM_ENABLE_PROJECTS=libc is deprecated. Please use " + message(WARNING "Using LLVM_ENABLE_PROJECTS=libc is deprecated now, and will " + "become a fatal error in a future release. Please use " "-DLLVM_ENABLE_RUNTIMES=libc or see the instructions at " "https://libc.llvm.org/ for building the runtimes.") endif() if ("compiler-rt" IN_LIST LLVM_ENABLE_PROJECTS) message(WARNING "Using LLVM_ENABLE_PROJECTS=compiler-rt is deprecated now, and will " - "become a fatal error in the LLVM 21 release. Please use " + "become a fatal error in a future release. Please use " "-DLLVM_ENABLE_RUNTIMES=compiler-rt or see the instructions at " "https://compiler-rt.llvm.org/ for building the runtimes.") endif() if ("offload" IN_LIST LLVM_ENABLE_PROJECTS) message(WARNING "Using LLVM_ENABLE_PROJECTS=offload is deprecated now, and will " - "become a fatal error in the LLVM 21 release. Please use " + "become a fatal error in a future release. Please use " "-DLLVM_ENABLE_RUNTIMES=offload or see the instructions at " "https://openmp.llvm.org/ for building the runtimes.") endif() if ("openmp" IN_LIST LLVM_ENABLE_PROJECTS) message(WARNING "Using LLVM_ENABLE_PROJECTS=openmp is deprecated now, and will " - "become a fatal error in the LLVM 21 release. Please use " + "become a fatal error in a future release. Please use " "-DLLVM_ENABLE_RUNTIMES=openmp or see the instructions at " "https://openmp.llvm.org/ for building the runtimes.") endif() @@ -221,7 +222,7 @@ endif () if ("libclc" IN_LIST LLVM_ENABLE_PROJECTS) message(WARNING "Using LLVM_ENABLE_PROJECTS=libclc is deprecated now, and will " - "become a fatal error in the LLVM 21 release. Please use " + "become a fatal error in a future release. Please use " "-DLLVM_ENABLE_RUNTIMES=libclc or see the instructions at " "https://libclc.llvm.org/ for building the runtimes.") endif() diff --git a/llvm/docs/NVPTXUsage.rst b/llvm/docs/NVPTXUsage.rst index d28eb68..2dc8f9f 100644 --- a/llvm/docs/NVPTXUsage.rst +++ b/llvm/docs/NVPTXUsage.rst @@ -971,6 +971,10 @@ Syntax: declare void @llvm.nvvm.prefetch.L1(ptr %ptr) declare void @llvm.nvvm.prefetch.L2(ptr %ptr) + declare void @llvm.nvvm.prefetch.tensormap.p0(ptr %ptr) + declare void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) %const_ptr) + declare void @llvm.nvvm.prefetch.tensormap.p101(ptr addrspace(101) %param_ptr) + declare void @llvm.nvvm.prefetch.global.L2.evict.normal(ptr addrspace(1) %global_ptr) declare void @llvm.nvvm.prefetch.global.L2.evict.last(ptr addrspace(1) %global_ptr) @@ -983,7 +987,10 @@ The '``@llvm.nvvm.prefetch.*``' and '``@llvm.nvvm.prefetchu.*``' intrinsic correspond to the '``prefetch.*``;' and '``prefetchu.*``' family of PTX instructions. The '``prefetch.*``' instructions bring the cache line containing the specified address in global or local memory address space into the -specified cache level (L1 or L2). The '`prefetchu.*``' instruction brings the cache line +specified cache level (L1 or L2). If the '``.tensormap``' qualifier is specified then the +prefetch instruction brings the cache line containing the specified address in the +'``.const``' or '``.param memory``' state space for subsequent use by the '``cp.async.bulk.tensor``' +instruction. The '`prefetchu.*``' instruction brings the cache line containing the specified generic address into the specified uniform cache level. If no address space is specified, it is assumed to be generic address. The intrinsic uses and eviction priority which can be accessed by the '``.level::eviction_priority``' modifier. diff --git a/llvm/include/llvm/Analysis/DependenceAnalysis.h b/llvm/include/llvm/Analysis/DependenceAnalysis.h index f98bd68..1679596 100644 --- a/llvm/include/llvm/Analysis/DependenceAnalysis.h +++ b/llvm/include/llvm/Analysis/DependenceAnalysis.h @@ -47,994 +47,908 @@ #include "llvm/Support/Compiler.h" namespace llvm { - class AAResults; - template <typename T> class ArrayRef; - class Loop; - class LoopInfo; - class SCEVConstant; - class raw_ostream; - - /// Dependence - This class represents a dependence between two memory - /// memory references in a function. It contains minimal information and - /// is used in the very common situation where the compiler is unable to - /// determine anything beyond the existence of a dependence; that is, it - /// represents a confused dependence (see also FullDependence). In most - /// cases (for output, flow, and anti dependences), the dependence implies - /// an ordering, where the source must precede the destination; in contrast, - /// input dependences are unordered. - /// - /// When a dependence graph is built, each Dependence will be a member of - /// the set of predecessor edges for its destination instruction and a set - /// if successor edges for its source instruction. These sets are represented - /// as singly-linked lists, with the "next" fields stored in the dependence - /// itelf. - class LLVM_ABI Dependence { - protected: - Dependence(Dependence &&) = default; - Dependence &operator=(Dependence &&) = default; - - public: - Dependence(Instruction *Source, Instruction *Destination, - const SCEVUnionPredicate &A) - : Src(Source), Dst(Destination), Assumptions(A) {} - virtual ~Dependence() = default; - - /// Dependence::DVEntry - Each level in the distance/direction vector - /// has a direction (or perhaps a union of several directions), and - /// perhaps a distance. - struct DVEntry { - enum : unsigned char { - NONE = 0, - LT = 1, - EQ = 2, - LE = 3, - GT = 4, - NE = 5, - GE = 6, - ALL = 7 - }; - unsigned char Direction : 3; // Init to ALL, then refine. - bool Scalar : 1; // Init to true. - bool PeelFirst : 1; // Peeling the first iteration will break dependence. - bool PeelLast : 1; // Peeling the last iteration will break the dependence. - bool Splitable : 1; // Splitting the loop will break dependence. - const SCEV *Distance = nullptr; // NULL implies no distance available. - DVEntry() - : Direction(ALL), Scalar(true), PeelFirst(false), PeelLast(false), - Splitable(false) {} +class AAResults; +template <typename T> class ArrayRef; +class Loop; +class LoopInfo; +class SCEVConstant; +class raw_ostream; + +/// Dependence - This class represents a dependence between two memory +/// memory references in a function. It contains minimal information and +/// is used in the very common situation where the compiler is unable to +/// determine anything beyond the existence of a dependence; that is, it +/// represents a confused dependence (see also FullDependence). In most +/// cases (for output, flow, and anti dependences), the dependence implies +/// an ordering, where the source must precede the destination; in contrast, +/// input dependences are unordered. +/// +/// When a dependence graph is built, each Dependence will be a member of +/// the set of predecessor edges for its destination instruction and a set +/// if successor edges for its source instruction. These sets are represented +/// as singly-linked lists, with the "next" fields stored in the dependence +/// itelf. +class LLVM_ABI Dependence { +protected: + Dependence(Dependence &&) = default; + Dependence &operator=(Dependence &&) = default; + +public: + Dependence(Instruction *Source, Instruction *Destination, + const SCEVUnionPredicate &A) + : Src(Source), Dst(Destination), Assumptions(A) {} + virtual ~Dependence() = default; + + /// Dependence::DVEntry - Each level in the distance/direction vector + /// has a direction (or perhaps a union of several directions), and + /// perhaps a distance. + struct DVEntry { + enum : unsigned char { + NONE = 0, + LT = 1, + EQ = 2, + LE = 3, + GT = 4, + NE = 5, + GE = 6, + ALL = 7 }; + unsigned char Direction : 3; // Init to ALL, then refine. + bool Scalar : 1; // Init to true. + bool PeelFirst : 1; // Peeling the first iteration will break dependence. + bool PeelLast : 1; // Peeling the last iteration will break the dependence. + bool Splitable : 1; // Splitting the loop will break dependence. + const SCEV *Distance = nullptr; // NULL implies no distance available. + DVEntry() + : Direction(ALL), Scalar(true), PeelFirst(false), PeelLast(false), + Splitable(false) {} + }; - /// getSrc - Returns the source instruction for this dependence. - /// - Instruction *getSrc() const { return Src; } - - /// getDst - Returns the destination instruction for this dependence. - /// - Instruction *getDst() const { return Dst; } - - /// isInput - Returns true if this is an input dependence. - /// - bool isInput() const; - - /// isOutput - Returns true if this is an output dependence. - /// - bool isOutput() const; - - /// isFlow - Returns true if this is a flow (aka true) dependence. - /// - bool isFlow() const; - - /// isAnti - Returns true if this is an anti dependence. - /// - bool isAnti() const; - - /// isOrdered - Returns true if dependence is Output, Flow, or Anti - /// - bool isOrdered() const { return isOutput() || isFlow() || isAnti(); } + /// getSrc - Returns the source instruction for this dependence. + Instruction *getSrc() const { return Src; } - /// isUnordered - Returns true if dependence is Input - /// - bool isUnordered() const { return isInput(); } + /// getDst - Returns the destination instruction for this dependence. + Instruction *getDst() const { return Dst; } - /// isLoopIndependent - Returns true if this is a loop-independent - /// dependence. - virtual bool isLoopIndependent() const { return true; } + /// isInput - Returns true if this is an input dependence. + bool isInput() const; - /// isConfused - Returns true if this dependence is confused - /// (the compiler understands nothing and makes worst-case - /// assumptions). - virtual bool isConfused() const { return true; } + /// isOutput - Returns true if this is an output dependence. + bool isOutput() const; - /// isConsistent - Returns true if this dependence is consistent - /// (occurs every time the source and destination are executed). - virtual bool isConsistent() const { return false; } + /// isFlow - Returns true if this is a flow (aka true) dependence. + bool isFlow() const; - /// getLevels - Returns the number of common loops surrounding the - /// source and destination of the dependence. - virtual unsigned getLevels() const { return 0; } + /// isAnti - Returns true if this is an anti dependence. + bool isAnti() const; - /// getDirection - Returns the direction associated with a particular - /// level. - virtual unsigned getDirection(unsigned Level) const { return DVEntry::ALL; } + /// isOrdered - Returns true if dependence is Output, Flow, or Anti + bool isOrdered() const { return isOutput() || isFlow() || isAnti(); } - /// getDistance - Returns the distance (or NULL) associated with a - /// particular level. - virtual const SCEV *getDistance(unsigned Level) const { return nullptr; } + /// isUnordered - Returns true if dependence is Input + bool isUnordered() const { return isInput(); } - /// Check if the direction vector is negative. A negative direction - /// vector means Src and Dst are reversed in the actual program. - virtual bool isDirectionNegative() const { return false; } + /// isLoopIndependent - Returns true if this is a loop-independent + /// dependence. + virtual bool isLoopIndependent() const { return true; } - /// If the direction vector is negative, normalize the direction - /// vector to make it non-negative. Normalization is done by reversing - /// Src and Dst, plus reversing the dependence directions and distances - /// in the vector. - virtual bool normalize(ScalarEvolution *SE) { return false; } + /// isConfused - Returns true if this dependence is confused + /// (the compiler understands nothing and makes worst-case assumptions). + virtual bool isConfused() const { return true; } - /// isPeelFirst - Returns true if peeling the first iteration from - /// this loop will break this dependence. - virtual bool isPeelFirst(unsigned Level) const { return false; } + /// isConsistent - Returns true if this dependence is consistent + /// (occurs every time the source and destination are executed). + virtual bool isConsistent() const { return false; } - /// isPeelLast - Returns true if peeling the last iteration from - /// this loop will break this dependence. - virtual bool isPeelLast(unsigned Level) const { return false; } + /// getLevels - Returns the number of common loops surrounding the + /// source and destination of the dependence. + virtual unsigned getLevels() const { return 0; } - /// isSplitable - Returns true if splitting this loop will break - /// the dependence. - virtual bool isSplitable(unsigned Level) const { return false; } + /// getDirection - Returns the direction associated with a particular level. + virtual unsigned getDirection(unsigned Level) const { return DVEntry::ALL; } - /// isScalar - Returns true if a particular level is scalar; that is, - /// if no subscript in the source or destination mention the induction - /// variable associated with the loop at this level. - virtual bool isScalar(unsigned Level) const; + /// getDistance - Returns the distance (or NULL) associated with a particular + /// level. + virtual const SCEV *getDistance(unsigned Level) const { return nullptr; } - /// getNextPredecessor - Returns the value of the NextPredecessor - /// field. - const Dependence *getNextPredecessor() const { return NextPredecessor; } + /// Check if the direction vector is negative. A negative direction + /// vector means Src and Dst are reversed in the actual program. + virtual bool isDirectionNegative() const { return false; } + + /// If the direction vector is negative, normalize the direction + /// vector to make it non-negative. Normalization is done by reversing + /// Src and Dst, plus reversing the dependence directions and distances + /// in the vector. + virtual bool normalize(ScalarEvolution *SE) { return false; } - /// getNextSuccessor - Returns the value of the NextSuccessor - /// field. - const Dependence *getNextSuccessor() const { return NextSuccessor; } + /// isPeelFirst - Returns true if peeling the first iteration from + /// this loop will break this dependence. + virtual bool isPeelFirst(unsigned Level) const { return false; } - /// setNextPredecessor - Sets the value of the NextPredecessor - /// field. - void setNextPredecessor(const Dependence *pred) { NextPredecessor = pred; } + /// isPeelLast - Returns true if peeling the last iteration from + /// this loop will break this dependence. + virtual bool isPeelLast(unsigned Level) const { return false; } - /// setNextSuccessor - Sets the value of the NextSuccessor - /// field. - void setNextSuccessor(const Dependence *succ) { NextSuccessor = succ; } + /// isSplitable - Returns true if splitting this loop will break the + /// dependence. + virtual bool isSplitable(unsigned Level) const { return false; } - /// getRuntimeAssumptions - Returns the runtime assumptions under which this - /// Dependence relation is valid. - SCEVUnionPredicate getRuntimeAssumptions() const { return Assumptions; } + /// isScalar - Returns true if a particular level is scalar; that is, + /// if no subscript in the source or destination mention the induction + /// variable associated with the loop at this level. + virtual bool isScalar(unsigned Level) const; + + /// getNextPredecessor - Returns the value of the NextPredecessor field. + const Dependence *getNextPredecessor() const { return NextPredecessor; } + + /// getNextSuccessor - Returns the value of the NextSuccessor field. + const Dependence *getNextSuccessor() const { return NextSuccessor; } + + /// setNextPredecessor - Sets the value of the NextPredecessor + /// field. + void setNextPredecessor(const Dependence *pred) { NextPredecessor = pred; } + + /// setNextSuccessor - Sets the value of the NextSuccessor field. + void setNextSuccessor(const Dependence *succ) { NextSuccessor = succ; } + + /// getRuntimeAssumptions - Returns the runtime assumptions under which this + /// Dependence relation is valid. + SCEVUnionPredicate getRuntimeAssumptions() const { return Assumptions; } + + /// dump - For debugging purposes, dumps a dependence to OS. + void dump(raw_ostream &OS) const; + +protected: + Instruction *Src, *Dst; + +private: + SCEVUnionPredicate Assumptions; + const Dependence *NextPredecessor = nullptr, *NextSuccessor = nullptr; + friend class DependenceInfo; +}; + +/// FullDependence - This class represents a dependence between two memory +/// references in a function. It contains detailed information about the +/// dependence (direction vectors, etc.) and is used when the compiler is +/// able to accurately analyze the interaction of the references; that is, +/// it is not a confused dependence (see Dependence). In most cases +/// (for output, flow, and anti dependences), the dependence implies an +/// ordering, where the source must precede the destination; in contrast, +/// input dependences are unordered. +class LLVM_ABI FullDependence final : public Dependence { +public: + FullDependence(Instruction *Source, Instruction *Destination, + const SCEVUnionPredicate &Assumes, + bool PossiblyLoopIndependent, unsigned Levels); + + /// isLoopIndependent - Returns true if this is a loop-independent + /// dependence. + bool isLoopIndependent() const override { return LoopIndependent; } + + /// isConfused - Returns true if this dependence is confused + /// (the compiler understands nothing and makes worst-case + /// assumptions). + bool isConfused() const override { return false; } + + /// isConsistent - Returns true if this dependence is consistent + /// (occurs every time the source and destination are executed). + bool isConsistent() const override { return Consistent; } + + /// getLevels - Returns the number of common loops surrounding the + /// source and destination of the dependence. + unsigned getLevels() const override { return Levels; } + + /// getDirection - Returns the direction associated with a particular + /// level. + unsigned getDirection(unsigned Level) const override; + + /// getDistance - Returns the distance (or NULL) associated with a + /// particular level. + const SCEV *getDistance(unsigned Level) const override; + + /// Check if the direction vector is negative. A negative direction + /// vector means Src and Dst are reversed in the actual program. + bool isDirectionNegative() const override; + + /// If the direction vector is negative, normalize the direction + /// vector to make it non-negative. Normalization is done by reversing + /// Src and Dst, plus reversing the dependence directions and distances + /// in the vector. + bool normalize(ScalarEvolution *SE) override; + + /// isPeelFirst - Returns true if peeling the first iteration from + /// this loop will break this dependence. + bool isPeelFirst(unsigned Level) const override; + + /// isPeelLast - Returns true if peeling the last iteration from + /// this loop will break this dependence. + bool isPeelLast(unsigned Level) const override; + + /// isSplitable - Returns true if splitting the loop will break + /// the dependence. + bool isSplitable(unsigned Level) const override; + + /// isScalar - Returns true if a particular level is scalar; that is, + /// if no subscript in the source or destination mention the induction + /// variable associated with the loop at this level. + bool isScalar(unsigned Level) const override; + +private: + unsigned short Levels; + bool LoopIndependent; + bool Consistent; // Init to true, then refine. + std::unique_ptr<DVEntry[]> DV; + friend class DependenceInfo; +}; + +/// DependenceInfo - This class is the main dependence-analysis driver. +class DependenceInfo { +public: + DependenceInfo(Function *F, AAResults *AA, ScalarEvolution *SE, LoopInfo *LI) + : AA(AA), SE(SE), LI(LI), F(F) {} + + /// Handle transitive invalidation when the cached analysis results go away. + LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA, + FunctionAnalysisManager::Invalidator &Inv); + + /// depends - Tests for a dependence between the Src and Dst instructions. + /// Returns NULL if no dependence; otherwise, returns a Dependence (or a + /// FullDependence) with as much information as can be gleaned. By default, + /// the dependence test collects a set of runtime assumptions that cannot be + /// solved at compilation time. By default UnderRuntimeAssumptions is false + /// for a safe approximation of the dependence relation that does not + /// require runtime checks. + LLVM_ABI std::unique_ptr<Dependence> + depends(Instruction *Src, Instruction *Dst, + bool UnderRuntimeAssumptions = false); + + /// getSplitIteration - Give a dependence that's splittable at some + /// particular level, return the iteration that should be used to split + /// the loop. + /// + /// Generally, the dependence analyzer will be used to build + /// a dependence graph for a function (basically a map from instructions + /// to dependences). Looking for cycles in the graph shows us loops + /// that cannot be trivially vectorized/parallelized. + /// + /// We can try to improve the situation by examining all the dependences + /// that make up the cycle, looking for ones we can break. + /// Sometimes, peeling the first or last iteration of a loop will break + /// dependences, and there are flags for those possibilities. + /// Sometimes, splitting a loop at some other iteration will do the trick, + /// and we've got a flag for that case. Rather than waste the space to + /// record the exact iteration (since we rarely know), we provide + /// a method that calculates the iteration. It's a drag that it must work + /// from scratch, but wonderful in that it's possible. + /// + /// Here's an example: + /// + /// for (i = 0; i < 10; i++) + /// A[i] = ... + /// ... = A[11 - i] + /// + /// There's a loop-carried flow dependence from the store to the load, + /// found by the weak-crossing SIV test. The dependence will have a flag, + /// indicating that the dependence can be broken by splitting the loop. + /// Calling getSplitIteration will return 5. + /// Splitting the loop breaks the dependence, like so: + /// + /// for (i = 0; i <= 5; i++) + /// A[i] = ... + /// ... = A[11 - i] + /// for (i = 6; i < 10; i++) + /// A[i] = ... + /// ... = A[11 - i] + /// + /// breaks the dependence and allows us to vectorize/parallelize + /// both loops. + LLVM_ABI const SCEV *getSplitIteration(const Dependence &Dep, unsigned Level); + + Function *getFunction() const { return F; } + + /// getRuntimeAssumptions - Returns all the runtime assumptions under which + /// the dependence test is valid. + LLVM_ABI SCEVUnionPredicate getRuntimeAssumptions() const; + +private: + AAResults *AA; + ScalarEvolution *SE; + LoopInfo *LI; + Function *F; + SmallVector<const SCEVPredicate *, 4> Assumptions; + + /// Subscript - This private struct represents a pair of subscripts from + /// a pair of potentially multi-dimensional array references. We use a + /// vector of them to guide subscript partitioning. + struct Subscript { + const SCEV *Src; + const SCEV *Dst; + enum ClassificationKind { ZIV, SIV, RDIV, MIV, NonLinear } Classification; + SmallBitVector Loops; + SmallBitVector GroupLoops; + SmallBitVector Group; + }; - /// dump - For debugging purposes, dumps a dependence to OS. - /// - void dump(raw_ostream &OS) const; + struct CoefficientInfo { + const SCEV *Coeff; + const SCEV *PosPart; + const SCEV *NegPart; + const SCEV *Iterations; + }; - protected: - Instruction *Src, *Dst; + struct BoundInfo { + const SCEV *Iterations; + const SCEV *Upper[8]; + const SCEV *Lower[8]; + unsigned char Direction; + unsigned char DirSet; + }; + /// Constraint - This private class represents a constraint, as defined + /// in the paper + /// + /// Practical Dependence Testing + /// Goff, Kennedy, Tseng + /// PLDI 1991 + /// + /// There are 5 kinds of constraint, in a hierarchy. + /// 1) Any - indicates no constraint, any dependence is possible. + /// 2) Line - A line ax + by = c, where a, b, and c are parameters, + /// representing the dependence equation. + /// 3) Distance - The value d of the dependence distance; + /// 4) Point - A point <x, y> representing the dependence from + /// iteration x to iteration y. + /// 5) Empty - No dependence is possible. + class Constraint { private: - SCEVUnionPredicate Assumptions; - const Dependence *NextPredecessor = nullptr, *NextSuccessor = nullptr; - friend class DependenceInfo; - }; + enum ConstraintKind { Empty, Point, Distance, Line, Any } Kind; + ScalarEvolution *SE; + const SCEV *A; + const SCEV *B; + const SCEV *C; + const Loop *AssociatedLoop; - /// FullDependence - This class represents a dependence between two memory - /// references in a function. It contains detailed information about the - /// dependence (direction vectors, etc.) and is used when the compiler is - /// able to accurately analyze the interaction of the references; that is, - /// it is not a confused dependence (see Dependence). In most cases - /// (for output, flow, and anti dependences), the dependence implies an - /// ordering, where the source must precede the destination; in contrast, - /// input dependences are unordered. - class LLVM_ABI FullDependence final : public Dependence { public: - FullDependence(Instruction *Source, Instruction *Destination, - const SCEVUnionPredicate &Assumes, - bool PossiblyLoopIndependent, unsigned Levels); - - /// isLoopIndependent - Returns true if this is a loop-independent - /// dependence. - bool isLoopIndependent() const override { return LoopIndependent; } - - /// isConfused - Returns true if this dependence is confused - /// (the compiler understands nothing and makes worst-case - /// assumptions). - bool isConfused() const override { return false; } - - /// isConsistent - Returns true if this dependence is consistent - /// (occurs every time the source and destination are executed). - bool isConsistent() const override { return Consistent; } - - /// getLevels - Returns the number of common loops surrounding the - /// source and destination of the dependence. - unsigned getLevels() const override { return Levels; } - - /// getDirection - Returns the direction associated with a particular - /// level. - unsigned getDirection(unsigned Level) const override; - - /// getDistance - Returns the distance (or NULL) associated with a - /// particular level. - const SCEV *getDistance(unsigned Level) const override; - - /// Check if the direction vector is negative. A negative direction - /// vector means Src and Dst are reversed in the actual program. - bool isDirectionNegative() const override; - - /// If the direction vector is negative, normalize the direction - /// vector to make it non-negative. Normalization is done by reversing - /// Src and Dst, plus reversing the dependence directions and distances - /// in the vector. - bool normalize(ScalarEvolution *SE) override; - - /// isPeelFirst - Returns true if peeling the first iteration from - /// this loop will break this dependence. - bool isPeelFirst(unsigned Level) const override; - - /// isPeelLast - Returns true if peeling the last iteration from - /// this loop will break this dependence. - bool isPeelLast(unsigned Level) const override; - - /// isSplitable - Returns true if splitting the loop will break - /// the dependence. - bool isSplitable(unsigned Level) const override; - - /// isScalar - Returns true if a particular level is scalar; that is, - /// if no subscript in the source or destination mention the induction - /// variable associated with the loop at this level. - bool isScalar(unsigned Level) const override; + /// isEmpty - Return true if the constraint is of kind Empty. + bool isEmpty() const { return Kind == Empty; } - private: - unsigned short Levels; - bool LoopIndependent; - bool Consistent; // Init to true, then refine. - std::unique_ptr<DVEntry[]> DV; - friend class DependenceInfo; - }; + /// isPoint - Return true if the constraint is of kind Point. + bool isPoint() const { return Kind == Point; } - /// DependenceInfo - This class is the main dependence-analysis driver. - /// - class DependenceInfo { - public: - DependenceInfo(Function *F, AAResults *AA, ScalarEvolution *SE, - LoopInfo *LI) - : AA(AA), SE(SE), LI(LI), F(F) {} - - /// Handle transitive invalidation when the cached analysis results go away. - LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA, - FunctionAnalysisManager::Invalidator &Inv); - - /// depends - Tests for a dependence between the Src and Dst instructions. - /// Returns NULL if no dependence; otherwise, returns a Dependence (or a - /// FullDependence) with as much information as can be gleaned. By default, - /// the dependence test collects a set of runtime assumptions that cannot be - /// solved at compilation time. By default UnderRuntimeAssumptions is false - /// for a safe approximation of the dependence relation that does not - /// require runtime checks. - LLVM_ABI std::unique_ptr<Dependence> - depends(Instruction *Src, Instruction *Dst, - bool UnderRuntimeAssumptions = false); - - /// getSplitIteration - Give a dependence that's splittable at some - /// particular level, return the iteration that should be used to split - /// the loop. - /// - /// Generally, the dependence analyzer will be used to build - /// a dependence graph for a function (basically a map from instructions - /// to dependences). Looking for cycles in the graph shows us loops - /// that cannot be trivially vectorized/parallelized. - /// - /// We can try to improve the situation by examining all the dependences - /// that make up the cycle, looking for ones we can break. - /// Sometimes, peeling the first or last iteration of a loop will break - /// dependences, and there are flags for those possibilities. - /// Sometimes, splitting a loop at some other iteration will do the trick, - /// and we've got a flag for that case. Rather than waste the space to - /// record the exact iteration (since we rarely know), we provide - /// a method that calculates the iteration. It's a drag that it must work - /// from scratch, but wonderful in that it's possible. - /// - /// Here's an example: - /// - /// for (i = 0; i < 10; i++) - /// A[i] = ... - /// ... = A[11 - i] - /// - /// There's a loop-carried flow dependence from the store to the load, - /// found by the weak-crossing SIV test. The dependence will have a flag, - /// indicating that the dependence can be broken by splitting the loop. - /// Calling getSplitIteration will return 5. - /// Splitting the loop breaks the dependence, like so: - /// - /// for (i = 0; i <= 5; i++) - /// A[i] = ... - /// ... = A[11 - i] - /// for (i = 6; i < 10; i++) - /// A[i] = ... - /// ... = A[11 - i] - /// - /// breaks the dependence and allows us to vectorize/parallelize - /// both loops. - LLVM_ABI const SCEV *getSplitIteration(const Dependence &Dep, - unsigned Level); - - Function *getFunction() const { return F; } - - /// getRuntimeAssumptions - Returns all the runtime assumptions under which - /// the dependence test is valid. - LLVM_ABI SCEVUnionPredicate getRuntimeAssumptions() const; + /// isDistance - Return true if the constraint is of kind Distance. + bool isDistance() const { return Kind == Distance; } - private: - AAResults *AA; - ScalarEvolution *SE; - LoopInfo *LI; - Function *F; - SmallVector<const SCEVPredicate *, 4> Assumptions; - - /// Subscript - This private struct represents a pair of subscripts from - /// a pair of potentially multi-dimensional array references. We use a - /// vector of them to guide subscript partitioning. - struct Subscript { - const SCEV *Src; - const SCEV *Dst; - enum ClassificationKind { ZIV, SIV, RDIV, MIV, NonLinear } Classification; - SmallBitVector Loops; - SmallBitVector GroupLoops; - SmallBitVector Group; - }; + /// isLine - Return true if the constraint is of kind Line. + /// Since Distance's can also be represented as Lines, we also return + /// true if the constraint is of kind Distance. + bool isLine() const { return Kind == Line || Kind == Distance; } - struct CoefficientInfo { - const SCEV *Coeff; - const SCEV *PosPart; - const SCEV *NegPart; - const SCEV *Iterations; - }; + /// isAny - Return true if the constraint is of kind Any; + bool isAny() const { return Kind == Any; } - struct BoundInfo { - const SCEV *Iterations; - const SCEV *Upper[8]; - const SCEV *Lower[8]; - unsigned char Direction; - unsigned char DirSet; - }; + /// getX - If constraint is a point <X, Y>, returns X. + /// Otherwise assert. + LLVM_ABI const SCEV *getX() const; - /// Constraint - This private class represents a constraint, as defined - /// in the paper - /// - /// Practical Dependence Testing - /// Goff, Kennedy, Tseng - /// PLDI 1991 - /// - /// There are 5 kinds of constraint, in a hierarchy. - /// 1) Any - indicates no constraint, any dependence is possible. - /// 2) Line - A line ax + by = c, where a, b, and c are parameters, - /// representing the dependence equation. - /// 3) Distance - The value d of the dependence distance; - /// 4) Point - A point <x, y> representing the dependence from - /// iteration x to iteration y. - /// 5) Empty - No dependence is possible. - class Constraint { - private: - enum ConstraintKind { Empty, Point, Distance, Line, Any } Kind; - ScalarEvolution *SE; - const SCEV *A; - const SCEV *B; - const SCEV *C; - const Loop *AssociatedLoop; - - public: - /// isEmpty - Return true if the constraint is of kind Empty. - bool isEmpty() const { return Kind == Empty; } - - /// isPoint - Return true if the constraint is of kind Point. - bool isPoint() const { return Kind == Point; } - - /// isDistance - Return true if the constraint is of kind Distance. - bool isDistance() const { return Kind == Distance; } - - /// isLine - Return true if the constraint is of kind Line. - /// Since Distance's can also be represented as Lines, we also return - /// true if the constraint is of kind Distance. - bool isLine() const { return Kind == Line || Kind == Distance; } - - /// isAny - Return true if the constraint is of kind Any; - bool isAny() const { return Kind == Any; } - - /// getX - If constraint is a point <X, Y>, returns X. - /// Otherwise assert. - LLVM_ABI const SCEV *getX() const; - - /// getY - If constraint is a point <X, Y>, returns Y. - /// Otherwise assert. - LLVM_ABI const SCEV *getY() const; - - /// getA - If constraint is a line AX + BY = C, returns A. - /// Otherwise assert. - LLVM_ABI const SCEV *getA() const; - - /// getB - If constraint is a line AX + BY = C, returns B. - /// Otherwise assert. - LLVM_ABI const SCEV *getB() const; - - /// getC - If constraint is a line AX + BY = C, returns C. - /// Otherwise assert. - LLVM_ABI const SCEV *getC() const; - - /// getD - If constraint is a distance, returns D. - /// Otherwise assert. - LLVM_ABI const SCEV *getD() const; - - /// getAssociatedLoop - Returns the loop associated with this constraint. - LLVM_ABI const Loop *getAssociatedLoop() const; - - /// setPoint - Change a constraint to Point. - LLVM_ABI void setPoint(const SCEV *X, const SCEV *Y, - const Loop *CurrentLoop); - - /// setLine - Change a constraint to Line. - LLVM_ABI void setLine(const SCEV *A, const SCEV *B, const SCEV *C, - const Loop *CurrentLoop); - - /// setDistance - Change a constraint to Distance. - LLVM_ABI void setDistance(const SCEV *D, const Loop *CurrentLoop); - - /// setEmpty - Change a constraint to Empty. - LLVM_ABI void setEmpty(); - - /// setAny - Change a constraint to Any. - LLVM_ABI void setAny(ScalarEvolution *SE); - - /// dump - For debugging purposes. Dumps the constraint - /// out to OS. - LLVM_ABI void dump(raw_ostream &OS) const; - }; + /// getY - If constraint is a point <X, Y>, returns Y. + /// Otherwise assert. + LLVM_ABI const SCEV *getY() const; - /// establishNestingLevels - Examines the loop nesting of the Src and Dst - /// instructions and establishes their shared loops. Sets the variables - /// CommonLevels, SrcLevels, and MaxLevels. - /// The source and destination instructions needn't be contained in the same - /// loop. The routine establishNestingLevels finds the level of most deeply - /// nested loop that contains them both, CommonLevels. An instruction that's - /// not contained in a loop is at level = 0. MaxLevels is equal to the level - /// of the source plus the level of the destination, minus CommonLevels. - /// This lets us allocate vectors MaxLevels in length, with room for every - /// distinct loop referenced in both the source and destination subscripts. - /// The variable SrcLevels is the nesting depth of the source instruction. - /// It's used to help calculate distinct loops referenced by the destination. - /// Here's the map from loops to levels: - /// 0 - unused - /// 1 - outermost common loop - /// ... - other common loops - /// CommonLevels - innermost common loop - /// ... - loops containing Src but not Dst - /// SrcLevels - innermost loop containing Src but not Dst - /// ... - loops containing Dst but not Src - /// MaxLevels - innermost loop containing Dst but not Src - /// Consider the follow code fragment: - /// for (a = ...) { - /// for (b = ...) { - /// for (c = ...) { - /// for (d = ...) { - /// A[] = ...; - /// } - /// } - /// for (e = ...) { - /// for (f = ...) { - /// for (g = ...) { - /// ... = A[]; - /// } - /// } - /// } - /// } - /// } - /// If we're looking at the possibility of a dependence between the store - /// to A (the Src) and the load from A (the Dst), we'll note that they - /// have 2 loops in common, so CommonLevels will equal 2 and the direction - /// vector for Result will have 2 entries. SrcLevels = 4 and MaxLevels = 7. - /// A map from loop names to level indices would look like - /// a - 1 - /// b - 2 = CommonLevels - /// c - 3 - /// d - 4 = SrcLevels - /// e - 5 - /// f - 6 - /// g - 7 = MaxLevels - void establishNestingLevels(const Instruction *Src, - const Instruction *Dst); - - unsigned CommonLevels, SrcLevels, MaxLevels; - - /// mapSrcLoop - Given one of the loops containing the source, return - /// its level index in our numbering scheme. - unsigned mapSrcLoop(const Loop *SrcLoop) const; - - /// mapDstLoop - Given one of the loops containing the destination, - /// return its level index in our numbering scheme. - unsigned mapDstLoop(const Loop *DstLoop) const; - - /// isLoopInvariant - Returns true if Expression is loop invariant - /// in LoopNest. - bool isLoopInvariant(const SCEV *Expression, const Loop *LoopNest) const; - - /// Makes sure all subscript pairs share the same integer type by - /// sign-extending as necessary. - /// Sign-extending a subscript is safe because getelementptr assumes the - /// array subscripts are signed. - void unifySubscriptType(ArrayRef<Subscript *> Pairs); - - /// removeMatchingExtensions - Examines a subscript pair. - /// If the source and destination are identically sign (or zero) - /// extended, it strips off the extension in an effort to - /// simplify the actual analysis. - void removeMatchingExtensions(Subscript *Pair); - - /// collectCommonLoops - Finds the set of loops from the LoopNest that - /// have a level <= CommonLevels and are referred to by the SCEV Expression. - void collectCommonLoops(const SCEV *Expression, - const Loop *LoopNest, - SmallBitVector &Loops) const; - - /// checkSrcSubscript - Examines the SCEV Src, returning true iff it's - /// linear. Collect the set of loops mentioned by Src. - bool checkSrcSubscript(const SCEV *Src, - const Loop *LoopNest, - SmallBitVector &Loops); - - /// checkDstSubscript - Examines the SCEV Dst, returning true iff it's - /// linear. Collect the set of loops mentioned by Dst. - bool checkDstSubscript(const SCEV *Dst, - const Loop *LoopNest, - SmallBitVector &Loops); - - /// isKnownPredicate - Compare X and Y using the predicate Pred. - /// Basically a wrapper for SCEV::isKnownPredicate, - /// but tries harder, especially in the presence of sign and zero - /// extensions and symbolics. - bool isKnownPredicate(ICmpInst::Predicate Pred, - const SCEV *X, - const SCEV *Y) const; - - /// isKnownLessThan - Compare to see if S is less than Size - /// Another wrapper for isKnownNegative(S - max(Size, 1)) with some extra - /// checking if S is an AddRec and we can prove lessthan using the loop - /// bounds. - bool isKnownLessThan(const SCEV *S, const SCEV *Size) const; - - /// isKnownNonNegative - Compare to see if S is known not to be negative - /// Uses the fact that S comes from Ptr, which may be an inbound GEP, - /// Proving there is no wrapping going on. - bool isKnownNonNegative(const SCEV *S, const Value *Ptr) const; - - /// collectUpperBound - All subscripts are the same type (on my machine, - /// an i64). The loop bound may be a smaller type. collectUpperBound - /// find the bound, if available, and zero extends it to the Type T. - /// (I zero extend since the bound should always be >= 0.) - /// If no upper bound is available, return NULL. - const SCEV *collectUpperBound(const Loop *l, Type *T) const; - - /// collectConstantUpperBound - Calls collectUpperBound(), then - /// attempts to cast it to SCEVConstant. If the cast fails, - /// returns NULL. - const SCEVConstant *collectConstantUpperBound(const Loop *l, Type *T) const; - - /// classifyPair - Examines the subscript pair (the Src and Dst SCEVs) - /// and classifies it as either ZIV, SIV, RDIV, MIV, or Nonlinear. - /// Collects the associated loops in a set. - Subscript::ClassificationKind classifyPair(const SCEV *Src, - const Loop *SrcLoopNest, - const SCEV *Dst, - const Loop *DstLoopNest, - SmallBitVector &Loops); - - /// testZIV - Tests the ZIV subscript pair (Src and Dst) for dependence. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// If the dependence isn't proven to exist, - /// marks the Result as inconsistent. - bool testZIV(const SCEV *Src, - const SCEV *Dst, - FullDependence &Result) const; - - /// testSIV - Tests the SIV subscript pair (Src and Dst) for dependence. - /// Things of the form [c1 + a1*i] and [c2 + a2*j], where - /// i and j are induction variables, c1 and c2 are loop invariant, - /// and a1 and a2 are constant. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Sets appropriate direction vector entry and, when possible, - /// the distance vector entry. - /// If the dependence isn't proven to exist, - /// marks the Result as inconsistent. - bool testSIV(const SCEV *Src, - const SCEV *Dst, - unsigned &Level, - FullDependence &Result, - Constraint &NewConstraint, - const SCEV *&SplitIter) const; - - /// testRDIV - Tests the RDIV subscript pair (Src and Dst) for dependence. - /// Things of the form [c1 + a1*i] and [c2 + a2*j] - /// where i and j are induction variables, c1 and c2 are loop invariant, - /// and a1 and a2 are constant. - /// With minor algebra, this test can also be used for things like - /// [c1 + a1*i + a2*j][c2]. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Marks the Result as inconsistent. - bool testRDIV(const SCEV *Src, - const SCEV *Dst, - FullDependence &Result) const; + /// getA - If constraint is a line AX + BY = C, returns A. + /// Otherwise assert. + LLVM_ABI const SCEV *getA() const; - /// testMIV - Tests the MIV subscript pair (Src and Dst) for dependence. - /// Returns true if dependence disproved. - /// Can sometimes refine direction vectors. - bool testMIV(const SCEV *Src, - const SCEV *Dst, - const SmallBitVector &Loops, - FullDependence &Result) const; - - /// strongSIVtest - Tests the strong SIV subscript pair (Src and Dst) - /// for dependence. - /// Things of the form [c1 + a*i] and [c2 + a*i], - /// where i is an induction variable, c1 and c2 are loop invariant, - /// and a is a constant - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Sets appropriate direction and distance. - bool strongSIVtest(const SCEV *Coeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *CurrentLoop, - unsigned Level, - FullDependence &Result, - Constraint &NewConstraint) const; - - /// weakCrossingSIVtest - Tests the weak-crossing SIV subscript pair - /// (Src and Dst) for dependence. - /// Things of the form [c1 + a*i] and [c2 - a*i], - /// where i is an induction variable, c1 and c2 are loop invariant, - /// and a is a constant. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Sets appropriate direction entry. - /// Set consistent to false. - /// Marks the dependence as splitable. - bool weakCrossingSIVtest(const SCEV *SrcCoeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *CurrentLoop, - unsigned Level, - FullDependence &Result, - Constraint &NewConstraint, - const SCEV *&SplitIter) const; - - /// ExactSIVtest - Tests the SIV subscript pair - /// (Src and Dst) for dependence. - /// Things of the form [c1 + a1*i] and [c2 + a2*i], - /// where i is an induction variable, c1 and c2 are loop invariant, - /// and a1 and a2 are constant. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Sets appropriate direction entry. - /// Set consistent to false. - bool exactSIVtest(const SCEV *SrcCoeff, - const SCEV *DstCoeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *CurrentLoop, - unsigned Level, - FullDependence &Result, - Constraint &NewConstraint) const; - - /// weakZeroSrcSIVtest - Tests the weak-zero SIV subscript pair - /// (Src and Dst) for dependence. - /// Things of the form [c1] and [c2 + a*i], - /// where i is an induction variable, c1 and c2 are loop invariant, - /// and a is a constant. See also weakZeroDstSIVtest. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Sets appropriate direction entry. - /// Set consistent to false. - /// If loop peeling will break the dependence, mark appropriately. - bool weakZeroSrcSIVtest(const SCEV *DstCoeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *CurrentLoop, - unsigned Level, - FullDependence &Result, - Constraint &NewConstraint) const; - - /// weakZeroDstSIVtest - Tests the weak-zero SIV subscript pair - /// (Src and Dst) for dependence. - /// Things of the form [c1 + a*i] and [c2], - /// where i is an induction variable, c1 and c2 are loop invariant, - /// and a is a constant. See also weakZeroSrcSIVtest. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Sets appropriate direction entry. - /// Set consistent to false. - /// If loop peeling will break the dependence, mark appropriately. - bool weakZeroDstSIVtest(const SCEV *SrcCoeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *CurrentLoop, - unsigned Level, - FullDependence &Result, - Constraint &NewConstraint) const; - - /// exactRDIVtest - Tests the RDIV subscript pair for dependence. - /// Things of the form [c1 + a*i] and [c2 + b*j], - /// where i and j are induction variable, c1 and c2 are loop invariant, - /// and a and b are constants. - /// Returns true if any possible dependence is disproved. - /// Marks the result as inconsistent. - /// Works in some cases that symbolicRDIVtest doesn't, - /// and vice versa. - bool exactRDIVtest(const SCEV *SrcCoeff, - const SCEV *DstCoeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *SrcLoop, - const Loop *DstLoop, - FullDependence &Result) const; + /// getB - If constraint is a line AX + BY = C, returns B. + /// Otherwise assert. + LLVM_ABI const SCEV *getB() const; - /// symbolicRDIVtest - Tests the RDIV subscript pair for dependence. - /// Things of the form [c1 + a*i] and [c2 + b*j], - /// where i and j are induction variable, c1 and c2 are loop invariant, - /// and a and b are constants. - /// Returns true if any possible dependence is disproved. - /// Marks the result as inconsistent. - /// Works in some cases that exactRDIVtest doesn't, - /// and vice versa. Can also be used as a backup for - /// ordinary SIV tests. - bool symbolicRDIVtest(const SCEV *SrcCoeff, - const SCEV *DstCoeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *SrcLoop, - const Loop *DstLoop) const; - - /// gcdMIVtest - Tests an MIV subscript pair for dependence. - /// Returns true if any possible dependence is disproved. - /// Marks the result as inconsistent. - /// Can sometimes disprove the equal direction for 1 or more loops. - // Can handle some symbolics that even the SIV tests don't get, - /// so we use it as a backup for everything. - bool gcdMIVtest(const SCEV *Src, - const SCEV *Dst, - FullDependence &Result) const; - - /// banerjeeMIVtest - Tests an MIV subscript pair for dependence. - /// Returns true if any possible dependence is disproved. - /// Marks the result as inconsistent. - /// Computes directions. - bool banerjeeMIVtest(const SCEV *Src, - const SCEV *Dst, - const SmallBitVector &Loops, - FullDependence &Result) const; - - /// collectCoefficientInfo - Walks through the subscript, - /// collecting each coefficient, the associated loop bounds, - /// and recording its positive and negative parts for later use. - CoefficientInfo *collectCoeffInfo(const SCEV *Subscript, - bool SrcFlag, - const SCEV *&Constant) const; - - /// getPositivePart - X^+ = max(X, 0). - /// - const SCEV *getPositivePart(const SCEV *X) const; - - /// getNegativePart - X^- = min(X, 0). - /// - const SCEV *getNegativePart(const SCEV *X) const; - - /// getLowerBound - Looks through all the bounds info and - /// computes the lower bound given the current direction settings - /// at each level. - const SCEV *getLowerBound(BoundInfo *Bound) const; - - /// getUpperBound - Looks through all the bounds info and - /// computes the upper bound given the current direction settings - /// at each level. - const SCEV *getUpperBound(BoundInfo *Bound) const; - - /// exploreDirections - Hierarchically expands the direction vector - /// search space, combining the directions of discovered dependences - /// in the DirSet field of Bound. Returns the number of distinct - /// dependences discovered. If the dependence is disproved, - /// it will return 0. - unsigned exploreDirections(unsigned Level, - CoefficientInfo *A, - CoefficientInfo *B, - BoundInfo *Bound, - const SmallBitVector &Loops, - unsigned &DepthExpanded, - const SCEV *Delta) const; - - /// testBounds - Returns true iff the current bounds are plausible. - bool testBounds(unsigned char DirKind, - unsigned Level, - BoundInfo *Bound, - const SCEV *Delta) const; - - /// findBoundsALL - Computes the upper and lower bounds for level K - /// using the * direction. Records them in Bound. - void findBoundsALL(CoefficientInfo *A, - CoefficientInfo *B, - BoundInfo *Bound, - unsigned K) const; - - /// findBoundsLT - Computes the upper and lower bounds for level K - /// using the < direction. Records them in Bound. - void findBoundsLT(CoefficientInfo *A, - CoefficientInfo *B, - BoundInfo *Bound, - unsigned K) const; - - /// findBoundsGT - Computes the upper and lower bounds for level K - /// using the > direction. Records them in Bound. - void findBoundsGT(CoefficientInfo *A, - CoefficientInfo *B, - BoundInfo *Bound, - unsigned K) const; - - /// findBoundsEQ - Computes the upper and lower bounds for level K - /// using the = direction. Records them in Bound. - void findBoundsEQ(CoefficientInfo *A, - CoefficientInfo *B, - BoundInfo *Bound, - unsigned K) const; - - /// intersectConstraints - Updates X with the intersection - /// of the Constraints X and Y. Returns true if X has changed. - bool intersectConstraints(Constraint *X, - const Constraint *Y); - - /// propagate - Review the constraints, looking for opportunities - /// to simplify a subscript pair (Src and Dst). - /// Return true if some simplification occurs. - /// If the simplification isn't exact (that is, if it is conservative - /// in terms of dependence), set consistent to false. - bool propagate(const SCEV *&Src, - const SCEV *&Dst, - SmallBitVector &Loops, - SmallVectorImpl<Constraint> &Constraints, - bool &Consistent); - - /// propagateDistance - Attempt to propagate a distance - /// constraint into a subscript pair (Src and Dst). - /// Return true if some simplification occurs. - /// If the simplification isn't exact (that is, if it is conservative - /// in terms of dependence), set consistent to false. - bool propagateDistance(const SCEV *&Src, - const SCEV *&Dst, - Constraint &CurConstraint, - bool &Consistent); - - /// propagatePoint - Attempt to propagate a point - /// constraint into a subscript pair (Src and Dst). - /// Return true if some simplification occurs. - bool propagatePoint(const SCEV *&Src, - const SCEV *&Dst, - Constraint &CurConstraint); - - /// propagateLine - Attempt to propagate a line - /// constraint into a subscript pair (Src and Dst). - /// Return true if some simplification occurs. - /// If the simplification isn't exact (that is, if it is conservative - /// in terms of dependence), set consistent to false. - bool propagateLine(const SCEV *&Src, - const SCEV *&Dst, - Constraint &CurConstraint, - bool &Consistent); - - /// findCoefficient - Given a linear SCEV, - /// return the coefficient corresponding to specified loop. - /// If there isn't one, return the SCEV constant 0. - /// For example, given a*i + b*j + c*k, returning the coefficient - /// corresponding to the j loop would yield b. - const SCEV *findCoefficient(const SCEV *Expr, - const Loop *TargetLoop) const; - - /// zeroCoefficient - Given a linear SCEV, - /// return the SCEV given by zeroing out the coefficient - /// corresponding to the specified loop. - /// For example, given a*i + b*j + c*k, zeroing the coefficient - /// corresponding to the j loop would yield a*i + c*k. - const SCEV *zeroCoefficient(const SCEV *Expr, - const Loop *TargetLoop) const; - - /// addToCoefficient - Given a linear SCEV Expr, - /// return the SCEV given by adding some Value to the - /// coefficient corresponding to the specified TargetLoop. - /// For example, given a*i + b*j + c*k, adding 1 to the coefficient - /// corresponding to the j loop would yield a*i + (b+1)*j + c*k. - const SCEV *addToCoefficient(const SCEV *Expr, - const Loop *TargetLoop, - const SCEV *Value) const; - - /// updateDirection - Update direction vector entry - /// based on the current constraint. - void updateDirection(Dependence::DVEntry &Level, - const Constraint &CurConstraint) const; - - /// Given a linear access function, tries to recover subscripts - /// for each dimension of the array element access. - bool tryDelinearize(Instruction *Src, Instruction *Dst, - SmallVectorImpl<Subscript> &Pair); - - /// Tries to delinearize \p Src and \p Dst access functions for a fixed size - /// multi-dimensional array. Calls tryDelinearizeFixedSizeImpl() to - /// delinearize \p Src and \p Dst separately, - bool tryDelinearizeFixedSize(Instruction *Src, Instruction *Dst, - const SCEV *SrcAccessFn, - const SCEV *DstAccessFn, - SmallVectorImpl<const SCEV *> &SrcSubscripts, - SmallVectorImpl<const SCEV *> &DstSubscripts); - - /// Tries to delinearize access function for a multi-dimensional array with - /// symbolic runtime sizes. - /// Returns true upon success and false otherwise. - bool tryDelinearizeParametricSize( - Instruction *Src, Instruction *Dst, const SCEV *SrcAccessFn, - const SCEV *DstAccessFn, SmallVectorImpl<const SCEV *> &SrcSubscripts, - SmallVectorImpl<const SCEV *> &DstSubscripts); - - /// checkSubscript - Helper function for checkSrcSubscript and - /// checkDstSubscript to avoid duplicate code - bool checkSubscript(const SCEV *Expr, const Loop *LoopNest, - SmallBitVector &Loops, bool IsSrc); - }; // class DependenceInfo - - /// AnalysisPass to compute dependence information in a function - class DependenceAnalysis : public AnalysisInfoMixin<DependenceAnalysis> { - public: - typedef DependenceInfo Result; - LLVM_ABI Result run(Function &F, FunctionAnalysisManager &FAM); + /// getC - If constraint is a line AX + BY = C, returns C. + /// Otherwise assert. + LLVM_ABI const SCEV *getC() const; - private: - LLVM_ABI static AnalysisKey Key; - friend struct AnalysisInfoMixin<DependenceAnalysis>; - }; // class DependenceAnalysis + /// getD - If constraint is a distance, returns D. + /// Otherwise assert. + LLVM_ABI const SCEV *getD() const; - /// Printer pass to dump DA results. - struct DependenceAnalysisPrinterPass - : public PassInfoMixin<DependenceAnalysisPrinterPass> { - DependenceAnalysisPrinterPass(raw_ostream &OS, - bool NormalizeResults = false) - : OS(OS), NormalizeResults(NormalizeResults) {} + /// getAssociatedLoop - Returns the loop associated with this constraint. + LLVM_ABI const Loop *getAssociatedLoop() const; - LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); + /// setPoint - Change a constraint to Point. + LLVM_ABI void setPoint(const SCEV *X, const SCEV *Y, + const Loop *CurrentLoop); - static bool isRequired() { return true; } + /// setLine - Change a constraint to Line. + LLVM_ABI void setLine(const SCEV *A, const SCEV *B, const SCEV *C, + const Loop *CurrentLoop); - private: - raw_ostream &OS; - bool NormalizeResults; - }; // class DependenceAnalysisPrinterPass + /// setDistance - Change a constraint to Distance. + LLVM_ABI void setDistance(const SCEV *D, const Loop *CurrentLoop); - /// Legacy pass manager pass to access dependence information - class LLVM_ABI DependenceAnalysisWrapperPass : public FunctionPass { - public: - static char ID; // Class identification, replacement for typeinfo - DependenceAnalysisWrapperPass(); + /// setEmpty - Change a constraint to Empty. + LLVM_ABI void setEmpty(); - bool runOnFunction(Function &F) override; - void releaseMemory() override; - void getAnalysisUsage(AnalysisUsage &) const override; - void print(raw_ostream &, const Module * = nullptr) const override; - DependenceInfo &getDI() const; + /// setAny - Change a constraint to Any. + LLVM_ABI void setAny(ScalarEvolution *SE); - private: - std::unique_ptr<DependenceInfo> info; - }; // class DependenceAnalysisWrapperPass + /// dump - For debugging purposes. Dumps the constraint + /// out to OS. + LLVM_ABI void dump(raw_ostream &OS) const; + }; + + /// establishNestingLevels - Examines the loop nesting of the Src and Dst + /// instructions and establishes their shared loops. Sets the variables + /// CommonLevels, SrcLevels, and MaxLevels. + /// The source and destination instructions needn't be contained in the same + /// loop. The routine establishNestingLevels finds the level of most deeply + /// nested loop that contains them both, CommonLevels. An instruction that's + /// not contained in a loop is at level = 0. MaxLevels is equal to the level + /// of the source plus the level of the destination, minus CommonLevels. + /// This lets us allocate vectors MaxLevels in length, with room for every + /// distinct loop referenced in both the source and destination subscripts. + /// The variable SrcLevels is the nesting depth of the source instruction. + /// It's used to help calculate distinct loops referenced by the destination. + /// Here's the map from loops to levels: + /// 0 - unused + /// 1 - outermost common loop + /// ... - other common loops + /// CommonLevels - innermost common loop + /// ... - loops containing Src but not Dst + /// SrcLevels - innermost loop containing Src but not Dst + /// ... - loops containing Dst but not Src + /// MaxLevels - innermost loop containing Dst but not Src + /// Consider the follow code fragment: + /// for (a = ...) { + /// for (b = ...) { + /// for (c = ...) { + /// for (d = ...) { + /// A[] = ...; + /// } + /// } + /// for (e = ...) { + /// for (f = ...) { + /// for (g = ...) { + /// ... = A[]; + /// } + /// } + /// } + /// } + /// } + /// If we're looking at the possibility of a dependence between the store + /// to A (the Src) and the load from A (the Dst), we'll note that they + /// have 2 loops in common, so CommonLevels will equal 2 and the direction + /// vector for Result will have 2 entries. SrcLevels = 4 and MaxLevels = 7. + /// A map from loop names to level indices would look like + /// a - 1 + /// b - 2 = CommonLevels + /// c - 3 + /// d - 4 = SrcLevels + /// e - 5 + /// f - 6 + /// g - 7 = MaxLevels + void establishNestingLevels(const Instruction *Src, const Instruction *Dst); + + unsigned CommonLevels, SrcLevels, MaxLevels; + + /// mapSrcLoop - Given one of the loops containing the source, return + /// its level index in our numbering scheme. + unsigned mapSrcLoop(const Loop *SrcLoop) const; + + /// mapDstLoop - Given one of the loops containing the destination, + /// return its level index in our numbering scheme. + unsigned mapDstLoop(const Loop *DstLoop) const; + + /// isLoopInvariant - Returns true if Expression is loop invariant + /// in LoopNest. + bool isLoopInvariant(const SCEV *Expression, const Loop *LoopNest) const; + + /// Makes sure all subscript pairs share the same integer type by + /// sign-extending as necessary. + /// Sign-extending a subscript is safe because getelementptr assumes the + /// array subscripts are signed. + void unifySubscriptType(ArrayRef<Subscript *> Pairs); + + /// removeMatchingExtensions - Examines a subscript pair. + /// If the source and destination are identically sign (or zero) + /// extended, it strips off the extension in an effort to + /// simplify the actual analysis. + void removeMatchingExtensions(Subscript *Pair); + + /// collectCommonLoops - Finds the set of loops from the LoopNest that + /// have a level <= CommonLevels and are referred to by the SCEV Expression. + void collectCommonLoops(const SCEV *Expression, const Loop *LoopNest, + SmallBitVector &Loops) const; + + /// checkSrcSubscript - Examines the SCEV Src, returning true iff it's + /// linear. Collect the set of loops mentioned by Src. + bool checkSrcSubscript(const SCEV *Src, const Loop *LoopNest, + SmallBitVector &Loops); + + /// checkDstSubscript - Examines the SCEV Dst, returning true iff it's + /// linear. Collect the set of loops mentioned by Dst. + bool checkDstSubscript(const SCEV *Dst, const Loop *LoopNest, + SmallBitVector &Loops); + + /// isKnownPredicate - Compare X and Y using the predicate Pred. + /// Basically a wrapper for SCEV::isKnownPredicate, + /// but tries harder, especially in the presence of sign and zero + /// extensions and symbolics. + bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *X, + const SCEV *Y) const; + + /// isKnownLessThan - Compare to see if S is less than Size + /// Another wrapper for isKnownNegative(S - max(Size, 1)) with some extra + /// checking if S is an AddRec and we can prove lessthan using the loop + /// bounds. + bool isKnownLessThan(const SCEV *S, const SCEV *Size) const; + + /// isKnownNonNegative - Compare to see if S is known not to be negative + /// Uses the fact that S comes from Ptr, which may be an inbound GEP, + /// Proving there is no wrapping going on. + bool isKnownNonNegative(const SCEV *S, const Value *Ptr) const; + + /// collectUpperBound - All subscripts are the same type (on my machine, + /// an i64). The loop bound may be a smaller type. collectUpperBound + /// find the bound, if available, and zero extends it to the Type T. + /// (I zero extend since the bound should always be >= 0.) + /// If no upper bound is available, return NULL. + const SCEV *collectUpperBound(const Loop *l, Type *T) const; + + /// collectConstantUpperBound - Calls collectUpperBound(), then + /// attempts to cast it to SCEVConstant. If the cast fails, + /// returns NULL. + const SCEVConstant *collectConstantUpperBound(const Loop *l, Type *T) const; + + /// classifyPair - Examines the subscript pair (the Src and Dst SCEVs) + /// and classifies it as either ZIV, SIV, RDIV, MIV, or Nonlinear. + /// Collects the associated loops in a set. + Subscript::ClassificationKind + classifyPair(const SCEV *Src, const Loop *SrcLoopNest, const SCEV *Dst, + const Loop *DstLoopNest, SmallBitVector &Loops); + + /// testZIV - Tests the ZIV subscript pair (Src and Dst) for dependence. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// If the dependence isn't proven to exist, + /// marks the Result as inconsistent. + bool testZIV(const SCEV *Src, const SCEV *Dst, FullDependence &Result) const; + + /// testSIV - Tests the SIV subscript pair (Src and Dst) for dependence. + /// Things of the form [c1 + a1*i] and [c2 + a2*j], where + /// i and j are induction variables, c1 and c2 are loop invariant, + /// and a1 and a2 are constant. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Sets appropriate direction vector entry and, when possible, + /// the distance vector entry. + /// If the dependence isn't proven to exist, + /// marks the Result as inconsistent. + bool testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level, + FullDependence &Result, Constraint &NewConstraint, + const SCEV *&SplitIter) const; + + /// testRDIV - Tests the RDIV subscript pair (Src and Dst) for dependence. + /// Things of the form [c1 + a1*i] and [c2 + a2*j] + /// where i and j are induction variables, c1 and c2 are loop invariant, + /// and a1 and a2 are constant. + /// With minor algebra, this test can also be used for things like + /// [c1 + a1*i + a2*j][c2]. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Marks the Result as inconsistent. + bool testRDIV(const SCEV *Src, const SCEV *Dst, FullDependence &Result) const; + + /// testMIV - Tests the MIV subscript pair (Src and Dst) for dependence. + /// Returns true if dependence disproved. + /// Can sometimes refine direction vectors. + bool testMIV(const SCEV *Src, const SCEV *Dst, const SmallBitVector &Loops, + FullDependence &Result) const; + + /// strongSIVtest - Tests the strong SIV subscript pair (Src and Dst) + /// for dependence. + /// Things of the form [c1 + a*i] and [c2 + a*i], + /// where i is an induction variable, c1 and c2 are loop invariant, + /// and a is a constant + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Sets appropriate direction and distance. + bool strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst, + const SCEV *DstConst, const Loop *CurrentLoop, + unsigned Level, FullDependence &Result, + Constraint &NewConstraint) const; + + /// weakCrossingSIVtest - Tests the weak-crossing SIV subscript pair + /// (Src and Dst) for dependence. + /// Things of the form [c1 + a*i] and [c2 - a*i], + /// where i is an induction variable, c1 and c2 are loop invariant, + /// and a is a constant. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Sets appropriate direction entry. + /// Set consistent to false. + /// Marks the dependence as splitable. + bool weakCrossingSIVtest(const SCEV *SrcCoeff, const SCEV *SrcConst, + const SCEV *DstConst, const Loop *CurrentLoop, + unsigned Level, FullDependence &Result, + Constraint &NewConstraint, + const SCEV *&SplitIter) const; + + /// ExactSIVtest - Tests the SIV subscript pair + /// (Src and Dst) for dependence. + /// Things of the form [c1 + a1*i] and [c2 + a2*i], + /// where i is an induction variable, c1 and c2 are loop invariant, + /// and a1 and a2 are constant. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Sets appropriate direction entry. + /// Set consistent to false. + bool exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff, + const SCEV *SrcConst, const SCEV *DstConst, + const Loop *CurrentLoop, unsigned Level, + FullDependence &Result, Constraint &NewConstraint) const; + + /// weakZeroSrcSIVtest - Tests the weak-zero SIV subscript pair + /// (Src and Dst) for dependence. + /// Things of the form [c1] and [c2 + a*i], + /// where i is an induction variable, c1 and c2 are loop invariant, + /// and a is a constant. See also weakZeroDstSIVtest. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Sets appropriate direction entry. + /// Set consistent to false. + /// If loop peeling will break the dependence, mark appropriately. + bool weakZeroSrcSIVtest(const SCEV *DstCoeff, const SCEV *SrcConst, + const SCEV *DstConst, const Loop *CurrentLoop, + unsigned Level, FullDependence &Result, + Constraint &NewConstraint) const; + + /// weakZeroDstSIVtest - Tests the weak-zero SIV subscript pair + /// (Src and Dst) for dependence. + /// Things of the form [c1 + a*i] and [c2], + /// where i is an induction variable, c1 and c2 are loop invariant, + /// and a is a constant. See also weakZeroSrcSIVtest. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Sets appropriate direction entry. + /// Set consistent to false. + /// If loop peeling will break the dependence, mark appropriately. + bool weakZeroDstSIVtest(const SCEV *SrcCoeff, const SCEV *SrcConst, + const SCEV *DstConst, const Loop *CurrentLoop, + unsigned Level, FullDependence &Result, + Constraint &NewConstraint) const; + + /// exactRDIVtest - Tests the RDIV subscript pair for dependence. + /// Things of the form [c1 + a*i] and [c2 + b*j], + /// where i and j are induction variable, c1 and c2 are loop invariant, + /// and a and b are constants. + /// Returns true if any possible dependence is disproved. + /// Marks the result as inconsistent. + /// Works in some cases that symbolicRDIVtest doesn't, + /// and vice versa. + bool exactRDIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff, + const SCEV *SrcConst, const SCEV *DstConst, + const Loop *SrcLoop, const Loop *DstLoop, + FullDependence &Result) const; + + /// symbolicRDIVtest - Tests the RDIV subscript pair for dependence. + /// Things of the form [c1 + a*i] and [c2 + b*j], + /// where i and j are induction variable, c1 and c2 are loop invariant, + /// and a and b are constants. + /// Returns true if any possible dependence is disproved. + /// Marks the result as inconsistent. + /// Works in some cases that exactRDIVtest doesn't, + /// and vice versa. Can also be used as a backup for + /// ordinary SIV tests. + bool symbolicRDIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff, + const SCEV *SrcConst, const SCEV *DstConst, + const Loop *SrcLoop, const Loop *DstLoop) const; + + /// gcdMIVtest - Tests an MIV subscript pair for dependence. + /// Returns true if any possible dependence is disproved. + /// Marks the result as inconsistent. + /// Can sometimes disprove the equal direction for 1 or more loops. + // Can handle some symbolics that even the SIV tests don't get, + /// so we use it as a backup for everything. + bool gcdMIVtest(const SCEV *Src, const SCEV *Dst, + FullDependence &Result) const; + + /// banerjeeMIVtest - Tests an MIV subscript pair for dependence. + /// Returns true if any possible dependence is disproved. + /// Marks the result as inconsistent. + /// Computes directions. + bool banerjeeMIVtest(const SCEV *Src, const SCEV *Dst, + const SmallBitVector &Loops, + FullDependence &Result) const; - /// createDependenceAnalysisPass - This creates an instance of the - /// DependenceAnalysis wrapper pass. - LLVM_ABI FunctionPass *createDependenceAnalysisWrapperPass(); + /// collectCoeffInfo - Walks through the subscript, collecting each + /// coefficient, the associated loop bounds, and recording its positive and + /// negative parts for later use. + CoefficientInfo *collectCoeffInfo(const SCEV *Subscript, bool SrcFlag, + const SCEV *&Constant) const; + + /// getPositivePart - X^+ = max(X, 0). + const SCEV *getPositivePart(const SCEV *X) const; + + /// getNegativePart - X^- = min(X, 0). + const SCEV *getNegativePart(const SCEV *X) const; + + /// getLowerBound - Looks through all the bounds info and + /// computes the lower bound given the current direction settings + /// at each level. + const SCEV *getLowerBound(BoundInfo *Bound) const; + + /// getUpperBound - Looks through all the bounds info and + /// computes the upper bound given the current direction settings + /// at each level. + const SCEV *getUpperBound(BoundInfo *Bound) const; + + /// exploreDirections - Hierarchically expands the direction vector + /// search space, combining the directions of discovered dependences + /// in the DirSet field of Bound. Returns the number of distinct + /// dependences discovered. If the dependence is disproved, + /// it will return 0. + unsigned exploreDirections(unsigned Level, CoefficientInfo *A, + CoefficientInfo *B, BoundInfo *Bound, + const SmallBitVector &Loops, + unsigned &DepthExpanded, const SCEV *Delta) const; + + /// testBounds - Returns true iff the current bounds are plausible. + bool testBounds(unsigned char DirKind, unsigned Level, BoundInfo *Bound, + const SCEV *Delta) const; + + /// findBoundsALL - Computes the upper and lower bounds for level K + /// using the * direction. Records them in Bound. + void findBoundsALL(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, + unsigned K) const; + + /// findBoundsLT - Computes the upper and lower bounds for level K + /// using the < direction. Records them in Bound. + void findBoundsLT(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, + unsigned K) const; + + /// findBoundsGT - Computes the upper and lower bounds for level K + /// using the > direction. Records them in Bound. + void findBoundsGT(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, + unsigned K) const; + + /// findBoundsEQ - Computes the upper and lower bounds for level K + /// using the = direction. Records them in Bound. + void findBoundsEQ(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, + unsigned K) const; + + /// intersectConstraints - Updates X with the intersection + /// of the Constraints X and Y. Returns true if X has changed. + bool intersectConstraints(Constraint *X, const Constraint *Y); + + /// propagate - Review the constraints, looking for opportunities + /// to simplify a subscript pair (Src and Dst). + /// Return true if some simplification occurs. + /// If the simplification isn't exact (that is, if it is conservative + /// in terms of dependence), set consistent to false. + bool propagate(const SCEV *&Src, const SCEV *&Dst, SmallBitVector &Loops, + SmallVectorImpl<Constraint> &Constraints, bool &Consistent); + + /// propagateDistance - Attempt to propagate a distance + /// constraint into a subscript pair (Src and Dst). + /// Return true if some simplification occurs. + /// If the simplification isn't exact (that is, if it is conservative + /// in terms of dependence), set consistent to false. + bool propagateDistance(const SCEV *&Src, const SCEV *&Dst, + Constraint &CurConstraint, bool &Consistent); + + /// propagatePoint - Attempt to propagate a point + /// constraint into a subscript pair (Src and Dst). + /// Return true if some simplification occurs. + bool propagatePoint(const SCEV *&Src, const SCEV *&Dst, + Constraint &CurConstraint); + + /// propagateLine - Attempt to propagate a line + /// constraint into a subscript pair (Src and Dst). + /// Return true if some simplification occurs. + /// If the simplification isn't exact (that is, if it is conservative + /// in terms of dependence), set consistent to false. + bool propagateLine(const SCEV *&Src, const SCEV *&Dst, + Constraint &CurConstraint, bool &Consistent); + + /// findCoefficient - Given a linear SCEV, + /// return the coefficient corresponding to specified loop. + /// If there isn't one, return the SCEV constant 0. + /// For example, given a*i + b*j + c*k, returning the coefficient + /// corresponding to the j loop would yield b. + const SCEV *findCoefficient(const SCEV *Expr, const Loop *TargetLoop) const; + + /// zeroCoefficient - Given a linear SCEV, + /// return the SCEV given by zeroing out the coefficient + /// corresponding to the specified loop. + /// For example, given a*i + b*j + c*k, zeroing the coefficient + /// corresponding to the j loop would yield a*i + c*k. + const SCEV *zeroCoefficient(const SCEV *Expr, const Loop *TargetLoop) const; + + /// addToCoefficient - Given a linear SCEV Expr, + /// return the SCEV given by adding some Value to the + /// coefficient corresponding to the specified TargetLoop. + /// For example, given a*i + b*j + c*k, adding 1 to the coefficient + /// corresponding to the j loop would yield a*i + (b+1)*j + c*k. + const SCEV *addToCoefficient(const SCEV *Expr, const Loop *TargetLoop, + const SCEV *Value) const; + + /// updateDirection - Update direction vector entry + /// based on the current constraint. + void updateDirection(Dependence::DVEntry &Level, + const Constraint &CurConstraint) const; + + /// Given a linear access function, tries to recover subscripts + /// for each dimension of the array element access. + bool tryDelinearize(Instruction *Src, Instruction *Dst, + SmallVectorImpl<Subscript> &Pair); + + /// Tries to delinearize \p Src and \p Dst access functions for a fixed size + /// multi-dimensional array. Calls tryDelinearizeFixedSizeImpl() to + /// delinearize \p Src and \p Dst separately, + bool tryDelinearizeFixedSize(Instruction *Src, Instruction *Dst, + const SCEV *SrcAccessFn, const SCEV *DstAccessFn, + SmallVectorImpl<const SCEV *> &SrcSubscripts, + SmallVectorImpl<const SCEV *> &DstSubscripts); + + /// Tries to delinearize access function for a multi-dimensional array with + /// symbolic runtime sizes. + /// Returns true upon success and false otherwise. + bool + tryDelinearizeParametricSize(Instruction *Src, Instruction *Dst, + const SCEV *SrcAccessFn, const SCEV *DstAccessFn, + SmallVectorImpl<const SCEV *> &SrcSubscripts, + SmallVectorImpl<const SCEV *> &DstSubscripts); + + /// checkSubscript - Helper function for checkSrcSubscript and + /// checkDstSubscript to avoid duplicate code + bool checkSubscript(const SCEV *Expr, const Loop *LoopNest, + SmallBitVector &Loops, bool IsSrc); +}; // class DependenceInfo + +/// AnalysisPass to compute dependence information in a function +class DependenceAnalysis : public AnalysisInfoMixin<DependenceAnalysis> { +public: + typedef DependenceInfo Result; + LLVM_ABI Result run(Function &F, FunctionAnalysisManager &FAM); + +private: + LLVM_ABI static AnalysisKey Key; + friend struct AnalysisInfoMixin<DependenceAnalysis>; +}; // class DependenceAnalysis + +/// Printer pass to dump DA results. +struct DependenceAnalysisPrinterPass + : public PassInfoMixin<DependenceAnalysisPrinterPass> { + DependenceAnalysisPrinterPass(raw_ostream &OS, bool NormalizeResults = false) + : OS(OS), NormalizeResults(NormalizeResults) {} + + LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); + + static bool isRequired() { return true; } + +private: + raw_ostream &OS; + bool NormalizeResults; +}; // class DependenceAnalysisPrinterPass + +/// Legacy pass manager pass to access dependence information +class LLVM_ABI DependenceAnalysisWrapperPass : public FunctionPass { +public: + static char ID; // Class identification, replacement for typeinfo + DependenceAnalysisWrapperPass(); + + bool runOnFunction(Function &F) override; + void releaseMemory() override; + void getAnalysisUsage(AnalysisUsage &) const override; + void print(raw_ostream &, const Module * = nullptr) const override; + DependenceInfo &getDI() const; + +private: + std::unique_ptr<DependenceInfo> info; +}; // class DependenceAnalysisWrapperPass + +/// createDependenceAnalysisPass - This creates an instance of the +/// DependenceAnalysis wrapper pass. +LLVM_ABI FunctionPass *createDependenceAnalysisWrapperPass(); } // namespace llvm diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h index 93b0a8d..ae86f35 100644 --- a/llvm/include/llvm/Analysis/InlineCost.h +++ b/llvm/include/llvm/Analysis/InlineCost.h @@ -336,7 +336,11 @@ LLVM_ABI std::optional<InlineCostFeatures> getInliningCostFeatures( ProfileSummaryInfo *PSI = nullptr, OptimizationRemarkEmitter *ORE = nullptr); -/// Minimal filter to detect invalid constructs for inlining. +/// Check if it is mechanically possible to inline the function \p Callee, based +/// on the contents of the function. +/// +/// See also \p CanInlineCallSite as an additional precondition necessary to +/// perform a valid inline in a particular use context. LLVM_ABI InlineResult isInlineViable(Function &Callee); // This pass is used to annotate instructions during the inline process for diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h index 75c0517..8d98255 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -50,14 +50,12 @@ public: struct BaseArgInfo { Type *Ty; SmallVector<ISD::ArgFlagsTy, 4> Flags; - bool IsFixed; BaseArgInfo(Type *Ty, - ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>(), - bool IsFixed = true) - : Ty(Ty), Flags(Flags), IsFixed(IsFixed) {} + ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>()) + : Ty(Ty), Flags(Flags) {} - BaseArgInfo() : Ty(nullptr), IsFixed(false) {} + BaseArgInfo() : Ty(nullptr) {} }; struct ArgInfo : public BaseArgInfo { @@ -81,8 +79,8 @@ public: ArgInfo(ArrayRef<Register> Regs, Type *Ty, unsigned OrigIndex, ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>(), - bool IsFixed = true, const Value *OrigValue = nullptr) - : BaseArgInfo(Ty, Flags, IsFixed), Regs(Regs), OrigValue(OrigValue), + const Value *OrigValue = nullptr) + : BaseArgInfo(Ty, Flags), Regs(Regs), OrigValue(OrigValue), OrigArgIndex(OrigIndex) { if (!Regs.empty() && Flags.empty()) this->Flags.push_back(ISD::ArgFlagsTy()); @@ -93,9 +91,8 @@ public: } ArgInfo(ArrayRef<Register> Regs, const Value &OrigValue, unsigned OrigIndex, - ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>(), - bool IsFixed = true) - : ArgInfo(Regs, OrigValue.getType(), OrigIndex, Flags, IsFixed, &OrigValue) {} + ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>()) + : ArgInfo(Regs, OrigValue.getType(), OrigIndex, Flags, &OrigValue) {} ArgInfo() = default; }; diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h index a28c7a9..ca76c04 100644 --- a/llvm/include/llvm/CodeGen/TargetCallingConv.h +++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h @@ -54,6 +54,8 @@ namespace ISD { unsigned IsInConsecutiveRegs : 1; unsigned IsCopyElisionCandidate : 1; ///< Argument copy elision candidate unsigned IsPointer : 1; + /// Whether this is part of a variable argument list (non-fixed). + unsigned IsVarArg : 1; unsigned ByValOrByRefSize = 0; ///< Byval or byref struct size @@ -67,7 +69,7 @@ namespace ISD { IsSwiftError(0), IsCFGuardTarget(0), IsHva(0), IsHvaStart(0), IsSecArgPass(0), MemAlign(0), OrigAlign(0), IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0), - IsCopyElisionCandidate(0), IsPointer(0) { + IsCopyElisionCandidate(0), IsPointer(0), IsVarArg(0) { static_assert(sizeof(*this) == 4 * sizeof(unsigned), "flags are too big"); } @@ -145,6 +147,9 @@ namespace ISD { bool isPointer() const { return IsPointer; } void setPointer() { IsPointer = 1; } + bool isVarArg() const { return IsVarArg; } + void setVarArg() { IsVarArg = 1; } + Align getNonZeroMemAlign() const { return decodeMaybeAlign(MemAlign).valueOrOne(); } @@ -239,9 +244,6 @@ namespace ISD { MVT VT; EVT ArgVT; - /// IsFixed - Is this a "fixed" value, ie not passed through a vararg "...". - bool IsFixed = false; - /// Index original Function's argument. unsigned OrigArgIndex; @@ -251,10 +253,9 @@ namespace ISD { unsigned PartOffset; OutputArg() = default; - OutputArg(ArgFlagsTy flags, MVT vt, EVT argvt, bool isfixed, - unsigned origIdx, unsigned partOffs) - : Flags(flags), IsFixed(isfixed), OrigArgIndex(origIdx), - PartOffset(partOffs) { + OutputArg(ArgFlagsTy flags, MVT vt, EVT argvt, unsigned origIdx, + unsigned partOffs) + : Flags(flags), OrigArgIndex(origIdx), PartOffset(partOffs) { VT = vt; ArgVT = argvt; } diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td index 967d166..1bcc442 100644 --- a/llvm/include/llvm/IR/IntrinsicsNVVM.td +++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td @@ -137,6 +137,7 @@ def llvm_global_ptr_ty : LLVMQualPointerType<1>; // (global)ptr def llvm_shared_ptr_ty : LLVMQualPointerType<3>; // (shared)ptr +def llvm_constant_ptr_ty: LLVMQualPointerType<4>; // (const)ptr def llvm_local_ptr_ty : LLVMQualPointerType<5>; // (local)ptr def llvm_tmem_ptr_ty : LLVMQualPointerType<6>; // (tensor memory)ptr def llvm_shared_cluster_ptr_ty : LLVMQualPointerType<7>; // (shared_cluster)ptr @@ -2212,15 +2213,17 @@ def int_nvvm_cp_async_bulk_tensor_prefetch_tile_gather4_2d // Intrinsics for Prefetch and Prefetchu let IntrProperties = [IntrArgMemOnly, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>] in { foreach level = ["L1", "L2"] in { - def int_nvvm_prefetch_ # level : Intrinsic<[], [llvm_ptr_ty]>; - def int_nvvm_prefetch_global_ # level : Intrinsic<[], [llvm_global_ptr_ty]>; - def int_nvvm_prefetch_local_ # level : Intrinsic<[], [llvm_local_ptr_ty]>; + def int_nvvm_prefetch_ # level : DefaultAttrsIntrinsic<[], [llvm_ptr_ty]>; + def int_nvvm_prefetch_global_ # level : DefaultAttrsIntrinsic<[], [llvm_global_ptr_ty]>; + def int_nvvm_prefetch_local_ # level : DefaultAttrsIntrinsic<[], [llvm_local_ptr_ty]>; } + def int_nvvm_prefetch_tensormap : DefaultAttrsIntrinsic<[], [llvm_anyptr_ty]>; + foreach eviction_priority = ["evict_normal", "evict_last"] in - def int_nvvm_prefetch_global_L2_ # eviction_priority : Intrinsic<[], [llvm_global_ptr_ty]>; + def int_nvvm_prefetch_global_L2_ # eviction_priority : DefaultAttrsIntrinsic<[], [llvm_global_ptr_ty]>; - def int_nvvm_prefetchu_L1 : Intrinsic<[], [llvm_ptr_ty]>; + def int_nvvm_prefetchu_L1 : DefaultAttrsIntrinsic<[], [llvm_ptr_ty]>; } // applypriority diff --git a/llvm/include/llvm/Support/GraphWriter.h b/llvm/include/llvm/Support/GraphWriter.h index 39a4c0b..af2e501 100644 --- a/llvm/include/llvm/Support/GraphWriter.h +++ b/llvm/include/llvm/Support/GraphWriter.h @@ -61,8 +61,7 @@ enum Name { LLVM_ABI bool DisplayGraph(StringRef Filename, bool wait = true, GraphProgram::Name program = GraphProgram::DOT); -template<typename GraphType> -class GraphWriter { +template <typename GraphType, typename Derived> class GraphWriterBase { raw_ostream &O; const GraphType &G; bool RenderUsingHTML = false; @@ -75,9 +74,15 @@ class GraphWriter { DOTTraits DTraits; static_assert(std::is_pointer_v<NodeRef>, - "FIXME: Currently GraphWriter requires the NodeRef type to be " - "a pointer.\nThe pointer usage should be moved to " - "DOTGraphTraits, and removed from GraphWriter itself."); + "FIXME: Currently GraphWriterBase requires the NodeRef type to " + "be a pointer.\nThe pointer usage should be moved to " + "DOTGraphTraits, and removed from GraphWriterBase itself."); + + // Cast the 'this' pointer to the derived type and return a reference. + Derived &getDerived() { return *static_cast<Derived *>(this); } + const Derived &getDerived() const { + return *static_cast<const Derived *>(this); + } // Writes the edge labels of the node to O and returns true if there are any // edge labels not equal to the empty string "". @@ -118,23 +123,24 @@ class GraphWriter { } public: - GraphWriter(raw_ostream &o, const GraphType &g, bool SN) : O(o), G(g) { + GraphWriterBase(raw_ostream &o, const GraphType &g, bool SN) : O(o), G(g) { DTraits = DOTTraits(SN); RenderUsingHTML = DTraits.renderNodesUsingHTML(); } + virtual ~GraphWriterBase() {} void writeGraph(const std::string &Title = "") { // Output the header for the graph... - writeHeader(Title); + getDerived().writeHeader(Title); // Emit all of the nodes in the graph... - writeNodes(); + getDerived().writeNodes(); // Output any customizations on the graph - DOTGraphTraits<GraphType>::addCustomGraphFeatures(G, *this); + DOTGraphTraits<GraphType>::addCustomGraphFeatures(G, getDerived()); // Output the end of the graph - writeFooter(); + getDerived().writeFooter(); } void writeHeader(const std::string &Title) { @@ -166,8 +172,8 @@ public: void writeNodes() { // Loop over the graph, printing it out... for (const auto Node : nodes<GraphType>(G)) - if (!isNodeHidden(Node)) - writeNode(Node); + if (!getDerived().isNodeHidden(Node)) + getDerived().writeNode(Node); } bool isNodeHidden(NodeRef Node) { return DTraits.isNodeHidden(Node, G); } @@ -302,9 +308,9 @@ public: if (DTraits.getEdgeSourceLabel(Node, EI).empty()) edgeidx = -1; - emitEdge(static_cast<const void*>(Node), edgeidx, - static_cast<const void*>(TargetNode), DestPort, - DTraits.getEdgeAttributes(Node, EI, G)); + getDerived().emitEdge(static_cast<const void *>(Node), edgeidx, + static_cast<const void *>(TargetNode), DestPort, + DTraits.getEdgeAttributes(Node, EI, G)); } } @@ -357,10 +363,17 @@ public: } }; -template<typename GraphType> +template <typename GraphType> +class GraphWriter : public GraphWriterBase<GraphType, GraphWriter<GraphType>> { +public: + GraphWriter(raw_ostream &o, const GraphType &g, bool SN) + : GraphWriterBase<GraphType, GraphWriter<GraphType>>(o, g, SN) {} + ~GraphWriter() override {} +}; + +template <typename GraphType> raw_ostream &WriteGraph(raw_ostream &O, const GraphType &G, - bool ShortNames = false, - const Twine &Title = "") { + bool ShortNames = false, const Twine &Title = "") { // Start the graph emission process... GraphWriter<GraphType> W(O, G, ShortNames); diff --git a/llvm/include/llvm/Target/TargetCallingConv.td b/llvm/include/llvm/Target/TargetCallingConv.td index d0533ca..75d8ae4 100644 --- a/llvm/include/llvm/Target/TargetCallingConv.td +++ b/llvm/include/llvm/Target/TargetCallingConv.td @@ -97,6 +97,12 @@ class CCIfVarArg<CCAction A> : CCIf<"State.isVarArg()", A> {} /// CCIfNotVarArg - If the current function is not vararg - apply the action class CCIfNotVarArg<CCAction A> : CCIf<"!State.isVarArg()", A> {} +/// Apply the action if argument is fixed (not vararg). +class CCIfArgFixed<CCAction A> : CCIf<"!ArgFlags.isVarArg()", A>; + +/// Apply the action if argument is vararg (not fixed). +class CCIfArgVarArg<CCAction A> : CCIf<"ArgFlags.isVarArg()", A>; + /// CCIfPtrAddrSpace - If the top-level parent of the current argument has /// pointer type in the specified address-space. class CCIfPtrAddrSpace<int AS, CCAction A> diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h index 6b56230..cfa06a5 100644 --- a/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -279,6 +279,9 @@ public: /// `InlinedCalls` above is used. SmallVector<CallBase *, 8> InlinedCallSites; + Value *ConvergenceControlToken = nullptr; + Instruction *CallSiteEHPad = nullptr; + /// Update profile for callee as well as cloned version. We need to do this /// for regular inlining, but not for inlining from sample profile loader. bool UpdateProfile; @@ -287,9 +290,36 @@ public: StaticAllocas.clear(); InlinedCalls.clear(); InlinedCallSites.clear(); + ConvergenceControlToken = nullptr; + CallSiteEHPad = nullptr; } }; +/// Check if it is legal to perform inlining of the function called by \p CB +/// into the caller at this particular use, and sets fields in \p IFI. +/// +/// This does not consider whether it is possible for the function callee itself +/// to be inlined; for that see isInlineViable. +LLVM_ABI InlineResult CanInlineCallSite(const CallBase &CB, + InlineFunctionInfo &IFI); + +/// This should generally not be used, use InlineFunction instead. +/// +/// Perform mechanical inlining of \p CB into the caller. +/// +/// This does not perform any legality or profitability checks for the +/// inlining. This assumes that CanInlineCallSite was already called, populated +/// \p IFI, and returned InlineResult::success. +/// +/// Also assumes that isInlineViable returned InlineResult::success for the +/// called function. +LLVM_ABI void InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI, + bool MergeAttributes = false, + AAResults *CalleeAAR = nullptr, + bool InsertLifetime = true, + Function *ForwardVarArgsTo = nullptr, + OptimizationRemarkEmitter *ORE = nullptr); + /// This function inlines the called function into the basic /// block of the caller. This returns false if it is not possible to inline /// this call. The program is still in a well defined state if this occurs @@ -328,7 +358,8 @@ LLVM_ABI InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes = false, AAResults *CalleeAAR = nullptr, bool InsertLifetime = true, - Function *ForwardVarArgsTo = nullptr); + Function *ForwardVarArgsTo = nullptr, + OptimizationRemarkEmitter *ORE = nullptr); /// Clones a loop \p OrigLoop. Returns the loop and the blocks in \p /// Blocks. diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp index f1473b2..256befa 100644 --- a/llvm/lib/Analysis/DependenceAnalysis.cpp +++ b/llvm/lib/Analysis/DependenceAnalysis.cpp @@ -180,8 +180,8 @@ static void dumpExampleDependence(raw_ostream &OS, DependenceInfo *DA, for (inst_iterator SrcI = inst_begin(F), SrcE = inst_end(F); SrcI != SrcE; ++SrcI) { if (SrcI->mayReadOrWriteMemory()) { - for (inst_iterator DstI = SrcI, DstE = inst_end(F); - DstI != DstE; ++DstI) { + for (inst_iterator DstI = SrcI, DstE = inst_end(F); DstI != DstE; + ++DstI) { if (DstI->mayReadOrWriteMemory()) { OS << "Src:" << *SrcI << " --> Dst:" << *DstI << "\n"; OS << " da analyze - "; @@ -203,7 +203,7 @@ static void dumpExampleDependence(raw_ostream &OS, DependenceInfo *DA, // Normalize negative direction vectors if required by clients. if (NormalizeResults && D->normalize(&SE)) - OS << "normalized - "; + OS << "normalized - "; D->dump(OS); for (unsigned Level = 1; Level <= D->getLevels(); Level++) { if (D->isSplitable(Level)) { @@ -227,8 +227,8 @@ static void dumpExampleDependence(raw_ostream &OS, DependenceInfo *DA, void DependenceAnalysisWrapperPass::print(raw_ostream &OS, const Module *) const { - dumpExampleDependence(OS, info.get(), - getAnalysis<ScalarEvolutionWrapperPass>().getSE(), false); + dumpExampleDependence( + OS, info.get(), getAnalysis<ScalarEvolutionWrapperPass>().getSE(), false); } PreservedAnalyses @@ -249,33 +249,26 @@ bool Dependence::isInput() const { return Src->mayReadFromMemory() && Dst->mayReadFromMemory(); } - // Returns true if this is an output dependence. bool Dependence::isOutput() const { return Src->mayWriteToMemory() && Dst->mayWriteToMemory(); } - // Returns true if this is an flow (aka true) dependence. bool Dependence::isFlow() const { return Src->mayWriteToMemory() && Dst->mayReadFromMemory(); } - // Returns true if this is an anti dependence. bool Dependence::isAnti() const { return Src->mayReadFromMemory() && Dst->mayWriteToMemory(); } - // Returns true if a particular level is scalar; that is, // if no subscript in the source or destination mention the induction // variable associated with the loop at this level. // Leave this out of line, so it will serve as a virtual method anchor -bool Dependence::isScalar(unsigned level) const { - return false; -} - +bool Dependence::isScalar(unsigned level) const { return false; } //===----------------------------------------------------------------------===// // FullDependence methods @@ -338,8 +331,7 @@ bool FullDependence::normalize(ScalarEvolution *SE) { DV[Level - 1].Direction = RevDirection; // Reverse the dependence distance as well. if (DV[Level - 1].Distance != nullptr) - DV[Level - 1].Distance = - SE->getNegativeSCEV(DV[Level - 1].Distance); + DV[Level - 1].Distance = SE->getNegativeSCEV(DV[Level - 1].Distance); } LLVM_DEBUG(dbgs() << "After normalizing negative direction vectors:\n"; @@ -355,14 +347,12 @@ unsigned FullDependence::getDirection(unsigned Level) const { return DV[Level - 1].Direction; } - // Returns the distance (or NULL) associated with a particular level. const SCEV *FullDependence::getDistance(unsigned Level) const { assert(0 < Level && Level <= Levels && "Level out of range"); return DV[Level - 1].Distance; } - // Returns true if a particular level is scalar; that is, // if no subscript in the source or destination mention the induction // variable associated with the loop at this level. @@ -371,7 +361,6 @@ bool FullDependence::isScalar(unsigned Level) const { return DV[Level - 1].Scalar; } - // Returns true if peeling the first iteration from this loop // will break this dependence. bool FullDependence::isPeelFirst(unsigned Level) const { @@ -379,7 +368,6 @@ bool FullDependence::isPeelFirst(unsigned Level) const { return DV[Level - 1].PeelFirst; } - // Returns true if peeling the last iteration from this loop // will break this dependence. bool FullDependence::isPeelLast(unsigned Level) const { @@ -387,14 +375,12 @@ bool FullDependence::isPeelLast(unsigned Level) const { return DV[Level - 1].PeelLast; } - // Returns true if splitting this loop will break the dependence. bool FullDependence::isSplitable(unsigned Level) const { assert(0 < Level && Level <= Levels && "Level out of range"); return DV[Level - 1].Splitable; } - //===----------------------------------------------------------------------===// // DependenceInfo::Constraint methods @@ -405,7 +391,6 @@ const SCEV *DependenceInfo::Constraint::getX() const { return A; } - // If constraint is a point <X, Y>, returns Y. // Otherwise assert. const SCEV *DependenceInfo::Constraint::getY() const { @@ -413,7 +398,6 @@ const SCEV *DependenceInfo::Constraint::getY() const { return B; } - // If constraint is a line AX + BY = C, returns A. // Otherwise assert. const SCEV *DependenceInfo::Constraint::getA() const { @@ -422,7 +406,6 @@ const SCEV *DependenceInfo::Constraint::getA() const { return A; } - // If constraint is a line AX + BY = C, returns B. // Otherwise assert. const SCEV *DependenceInfo::Constraint::getB() const { @@ -431,7 +414,6 @@ const SCEV *DependenceInfo::Constraint::getB() const { return B; } - // If constraint is a line AX + BY = C, returns C. // Otherwise assert. const SCEV *DependenceInfo::Constraint::getC() const { @@ -440,7 +422,6 @@ const SCEV *DependenceInfo::Constraint::getC() const { return C; } - // If constraint is a distance, returns D. // Otherwise assert. const SCEV *DependenceInfo::Constraint::getD() const { @@ -448,7 +429,6 @@ const SCEV *DependenceInfo::Constraint::getD() const { return SE->getNegativeSCEV(C); } - // Returns the loop associated with this constraint. const Loop *DependenceInfo::Constraint::getAssociatedLoop() const { assert((Kind == Distance || Kind == Line || Kind == Point) && @@ -499,17 +479,16 @@ LLVM_DUMP_METHOD void DependenceInfo::Constraint::dump(raw_ostream &OS) const { else if (isPoint()) OS << " Point is <" << *getX() << ", " << *getY() << ">\n"; else if (isDistance()) - OS << " Distance is " << *getD() << - " (" << *getA() << "*X + " << *getB() << "*Y = " << *getC() << ")\n"; + OS << " Distance is " << *getD() << " (" << *getA() << "*X + " << *getB() + << "*Y = " << *getC() << ")\n"; else if (isLine()) - OS << " Line is " << *getA() << "*X + " << - *getB() << "*Y = " << *getC() << "\n"; + OS << " Line is " << *getA() << "*X + " << *getB() << "*Y = " << *getC() + << "\n"; else llvm_unreachable("unknown constraint type in Constraint::dump"); } #endif - // Updates X with the intersection // of the Constraints X and Y. Returns true if X has changed. // Corresponds to Figure 4 from the paper @@ -591,15 +570,14 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) { const SCEV *A1B2 = SE->getMulExpr(X->getA(), Y->getB()); const SCEV *A2B1 = SE->getMulExpr(Y->getA(), X->getB()); const SCEVConstant *C1A2_C2A1 = - dyn_cast<SCEVConstant>(SE->getMinusSCEV(C1A2, C2A1)); + dyn_cast<SCEVConstant>(SE->getMinusSCEV(C1A2, C2A1)); const SCEVConstant *C1B2_C2B1 = - dyn_cast<SCEVConstant>(SE->getMinusSCEV(C1B2, C2B1)); + dyn_cast<SCEVConstant>(SE->getMinusSCEV(C1B2, C2B1)); const SCEVConstant *A1B2_A2B1 = - dyn_cast<SCEVConstant>(SE->getMinusSCEV(A1B2, A2B1)); + dyn_cast<SCEVConstant>(SE->getMinusSCEV(A1B2, A2B1)); const SCEVConstant *A2B1_A1B2 = - dyn_cast<SCEVConstant>(SE->getMinusSCEV(A2B1, A1B2)); - if (!C1B2_C2B1 || !C1A2_C2A1 || - !A1B2_A2B1 || !A2B1_A1B2) + dyn_cast<SCEVConstant>(SE->getMinusSCEV(A2B1, A1B2)); + if (!C1B2_C2B1 || !C1A2_C2A1 || !A1B2_A2B1 || !A2B1_A1B2) return false; APInt Xtop = C1B2_C2B1->getAPInt(); APInt Xbot = A1B2_A2B1->getAPInt(); @@ -626,8 +604,8 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) { ++DeltaSuccesses; return true; } - if (const SCEVConstant *CUB = - collectConstantUpperBound(X->getAssociatedLoop(), Prod1->getType())) { + if (const SCEVConstant *CUB = collectConstantUpperBound( + X->getAssociatedLoop(), Prod1->getType())) { const APInt &UpperBound = CUB->getAPInt(); LLVM_DEBUG(dbgs() << "\t\tupper bound = " << UpperBound << "\n"); if (Xq.sgt(UpperBound) || Yq.sgt(UpperBound)) { @@ -636,8 +614,7 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) { return true; } } - X->setPoint(SE->getConstant(Xq), - SE->getConstant(Yq), + X->setPoint(SE->getConstant(Xq), SE->getConstant(Yq), X->getAssociatedLoop()); ++DeltaSuccesses; return true; @@ -667,7 +644,6 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) { return false; } - //===----------------------------------------------------------------------===// // DependenceInfo methods @@ -737,8 +713,7 @@ void Dependence::dump(raw_ostream &OS) const { // tbaa, non-overlapping regions etc), then it is known there is no dependecy. // Otherwise the underlying objects are checked to see if they point to // different identifiable objects. -static AliasResult underlyingObjectsAlias(AAResults *AA, - const DataLayout &DL, +static AliasResult underlyingObjectsAlias(AAResults *AA, const DataLayout &DL, const MemoryLocation &LocA, const MemoryLocation &LocB) { // Check the original locations (minus size) for noalias, which can happen for @@ -773,8 +748,7 @@ static AliasResult underlyingObjectsAlias(AAResults *AA, // Returns true if the load or store can be analyzed. Atomic and volatile // operations have properties which this analysis does not understand. -static -bool isLoadOrStore(const Instruction *I) { +static bool isLoadOrStore(const Instruction *I) { if (const LoadInst *LI = dyn_cast<LoadInst>(I)) return LI->isUnordered(); else if (const StoreInst *SI = dyn_cast<StoreInst>(I)) @@ -782,7 +756,6 @@ bool isLoadOrStore(const Instruction *I) { return false; } - // Examines the loop nesting of the Src and Dst // instructions and establishes their shared loops. Sets the variables // CommonLevels, SrcLevels, and MaxLevels. @@ -860,14 +833,12 @@ void DependenceInfo::establishNestingLevels(const Instruction *Src, MaxLevels -= CommonLevels; } - // Given one of the loops containing the source, return // its level index in our numbering scheme. unsigned DependenceInfo::mapSrcLoop(const Loop *SrcLoop) const { return SrcLoop->getLoopDepth(); } - // Given one of the loops containing the destination, // return its level index in our numbering scheme. unsigned DependenceInfo::mapDstLoop(const Loop *DstLoop) const { @@ -880,7 +851,6 @@ unsigned DependenceInfo::mapDstLoop(const Loop *DstLoop) const { return D; } - // Returns true if Expression is loop invariant in LoopNest. bool DependenceInfo::isLoopInvariant(const SCEV *Expression, const Loop *LoopNest) const { @@ -896,8 +866,6 @@ bool DependenceInfo::isLoopInvariant(const SCEV *Expression, return SE->isLoopInvariant(Expression, LoopNest->getOutermostLoop()); } - - // Finds the set of loops from the LoopNest that // have a level <= CommonLevels and are referred to by the SCEV Expression. void DependenceInfo::collectCommonLoops(const SCEV *Expression, @@ -924,9 +892,9 @@ void DependenceInfo::unifySubscriptType(ArrayRef<Subscript *> Pairs) { IntegerType *SrcTy = dyn_cast<IntegerType>(Src->getType()); IntegerType *DstTy = dyn_cast<IntegerType>(Dst->getType()); if (SrcTy == nullptr || DstTy == nullptr) { - assert(SrcTy == DstTy && "This function only unify integer types and " - "expect Src and Dst share the same type " - "otherwise."); + assert(SrcTy == DstTy && + "This function only unify integer types and " + "expect Src and Dst share the same type otherwise."); continue; } if (SrcTy->getBitWidth() > widestWidthSeen) { @@ -939,7 +907,6 @@ void DependenceInfo::unifySubscriptType(ArrayRef<Subscript *> Pairs) { } } - assert(widestWidthSeen > 0); // Now extend each pair to the widest seen. @@ -949,9 +916,9 @@ void DependenceInfo::unifySubscriptType(ArrayRef<Subscript *> Pairs) { IntegerType *SrcTy = dyn_cast<IntegerType>(Src->getType()); IntegerType *DstTy = dyn_cast<IntegerType>(Dst->getType()); if (SrcTy == nullptr || DstTy == nullptr) { - assert(SrcTy == DstTy && "This function only unify integer types and " - "expect Src and Dst share the same type " - "otherwise."); + assert(SrcTy == DstTy && + "This function only unify integer types and " + "expect Src and Dst share the same type otherwise."); continue; } if (SrcTy->getBitWidth() < widestWidthSeen) @@ -1028,7 +995,6 @@ bool DependenceInfo::checkDstSubscript(const SCEV *Dst, const Loop *LoopNest, return checkSubscript(Dst, LoopNest, Loops, false); } - // Examines the subscript pair (the Src and Dst SCEVs) // and classifies it as either ZIV, SIV, RDIV, MIV, or Nonlinear. // Collects the associated loops in a set. @@ -1049,14 +1015,12 @@ DependenceInfo::classifyPair(const SCEV *Src, const Loop *SrcLoopNest, return Subscript::ZIV; if (N == 1) return Subscript::SIV; - if (N == 2 && (SrcLoops.count() == 0 || - DstLoops.count() == 0 || + if (N == 2 && (SrcLoops.count() == 0 || DstLoops.count() == 0 || (SrcLoops.count() == 1 && DstLoops.count() == 1))) return Subscript::RDIV; return Subscript::MIV; } - // A wrapper around SCEV::isKnownPredicate. // Looks for cases where we're interested in comparing for equality. // If both X and Y have been identically sign or zero extended, @@ -1069,12 +1033,9 @@ DependenceInfo::classifyPair(const SCEV *Src, const Loop *SrcLoopNest, // involving symbolics. bool DependenceInfo::isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *X, const SCEV *Y) const { - if (Pred == CmpInst::ICMP_EQ || - Pred == CmpInst::ICMP_NE) { - if ((isa<SCEVSignExtendExpr>(X) && - isa<SCEVSignExtendExpr>(Y)) || - (isa<SCEVZeroExtendExpr>(X) && - isa<SCEVZeroExtendExpr>(Y))) { + if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) { + if ((isa<SCEVSignExtendExpr>(X) && isa<SCEVSignExtendExpr>(Y)) || + (isa<SCEVZeroExtendExpr>(X) && isa<SCEVZeroExtendExpr>(Y))) { const SCEVIntegralCastExpr *CX = cast<SCEVIntegralCastExpr>(X); const SCEVIntegralCastExpr *CY = cast<SCEVIntegralCastExpr>(Y); const SCEV *Xop = CX->getOperand(); @@ -1111,7 +1072,10 @@ bool DependenceInfo::isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *X, } } -/// Compare to see if S is less than Size, using isKnownNegative(S - max(Size, 1)) +/// Compare to see if S is less than Size, using +/// +/// isKnownNegative(S - max(Size, 1)) +/// /// with some extra checking if S is an AddRec and we can prove less-than using /// the loop bounds. bool DependenceInfo::isKnownLessThan(const SCEV *S, const SCEV *Size) const { @@ -1178,7 +1142,6 @@ const SCEV *DependenceInfo::collectUpperBound(const Loop *L, Type *T) const { return nullptr; } - // Calls collectUpperBound(), then attempts to cast it to SCEVConstant. // If the cast fails, returns NULL. const SCEVConstant *DependenceInfo::collectConstantUpperBound(const Loop *L, @@ -1188,7 +1151,6 @@ const SCEVConstant *DependenceInfo::collectConstantUpperBound(const Loop *L, return nullptr; } - // testZIV - // When we have a pair of subscripts of the form [c1] and [c2], // where c1 and c2 are both loop invariant, we attack it using @@ -1218,7 +1180,6 @@ bool DependenceInfo::testZIV(const SCEV *Src, const SCEV *Dst, return false; // possibly dependent } - // strongSIVtest - // From the paper, Practical Dependence Testing, Section 4.2.1 // @@ -1270,9 +1231,9 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst, LLVM_DEBUG(dbgs() << "\t UpperBound = " << *UpperBound); LLVM_DEBUG(dbgs() << ", " << *UpperBound->getType() << "\n"); const SCEV *AbsDelta = - SE->isKnownNonNegative(Delta) ? Delta : SE->getNegativeSCEV(Delta); + SE->isKnownNonNegative(Delta) ? Delta : SE->getNegativeSCEV(Delta); const SCEV *AbsCoeff = - SE->isKnownNonNegative(Coeff) ? Coeff : SE->getNegativeSCEV(Coeff); + SE->isKnownNonNegative(Coeff) ? Coeff : SE->getNegativeSCEV(Coeff); const SCEV *Product = SE->getMulExpr(UpperBound, AbsCoeff); if (isKnownPredicate(CmpInst::ICMP_SGT, AbsDelta, Product)) { // Distance greater than trip count - no dependence @@ -1286,7 +1247,7 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst, if (isa<SCEVConstant>(Delta) && isa<SCEVConstant>(Coeff)) { APInt ConstDelta = cast<SCEVConstant>(Delta)->getAPInt(); APInt ConstCoeff = cast<SCEVConstant>(Coeff)->getAPInt(); - APInt Distance = ConstDelta; // these need to be initialized + APInt Distance = ConstDelta; // these need to be initialized APInt Remainder = ConstDelta; APInt::sdivrem(ConstDelta, ConstCoeff, Distance, Remainder); LLVM_DEBUG(dbgs() << "\t Distance = " << Distance << "\n"); @@ -1307,29 +1268,25 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst, else Result.DV[Level].Direction &= Dependence::DVEntry::EQ; ++StrongSIVsuccesses; - } - else if (Delta->isZero()) { + } else if (Delta->isZero()) { // since 0/X == 0 Result.DV[Level].Distance = Delta; NewConstraint.setDistance(Delta, CurLoop); Result.DV[Level].Direction &= Dependence::DVEntry::EQ; ++StrongSIVsuccesses; - } - else { + } else { if (Coeff->isOne()) { LLVM_DEBUG(dbgs() << "\t Distance = " << *Delta << "\n"); Result.DV[Level].Distance = Delta; // since X/1 == X NewConstraint.setDistance(Delta, CurLoop); - } - else { + } else { Result.Consistent = false; - NewConstraint.setLine(Coeff, - SE->getNegativeSCEV(Coeff), + NewConstraint.setLine(Coeff, SE->getNegativeSCEV(Coeff), SE->getNegativeSCEV(Delta), CurLoop); } // maybe we can get a useful direction - bool DeltaMaybeZero = !SE->isKnownNonZero(Delta); + bool DeltaMaybeZero = !SE->isKnownNonZero(Delta); bool DeltaMaybePositive = !SE->isKnownNonPositive(Delta); bool DeltaMaybeNegative = !SE->isKnownNonNegative(Delta); bool CoeffMaybePositive = !SE->isKnownNonPositive(Coeff); @@ -1353,7 +1310,6 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst, return false; } - // weakCrossingSIVtest - // From the paper, Practical Dependence Testing, Section 4.2.2 // @@ -1447,8 +1403,8 @@ bool DependenceInfo::weakCrossingSIVtest( if (const SCEV *UpperBound = collectUpperBound(CurLoop, Delta->getType())) { LLVM_DEBUG(dbgs() << "\t UpperBound = " << *UpperBound << "\n"); const SCEV *ConstantTwo = SE->getConstant(UpperBound->getType(), 2); - const SCEV *ML = SE->getMulExpr(SE->getMulExpr(ConstCoeff, UpperBound), - ConstantTwo); + const SCEV *ML = + SE->getMulExpr(SE->getMulExpr(ConstCoeff, UpperBound), ConstantTwo); LLVM_DEBUG(dbgs() << "\t ML = " << *ML << "\n"); if (isKnownPredicate(CmpInst::ICMP_SGT, Delta, ML)) { // Delta too big, no dependence @@ -1498,7 +1454,6 @@ bool DependenceInfo::weakCrossingSIVtest( return false; } - // Kirch's algorithm, from // // Optimizing Supercompilers for Supercomputers @@ -1519,9 +1474,11 @@ static bool findGCD(unsigned Bits, const APInt &AM, const APInt &BM, APInt R = G0; APInt::sdivrem(G0, G1, Q, R); while (R != 0) { + // clang-format off APInt A2 = A0 - Q*A1; A0 = A1; A1 = A2; APInt B2 = B0 - Q*B1; B0 = B1; B1 = B2; G0 = G1; G1 = R; + // clang-format on APInt::sdivrem(G0, G1, Q, R); } G = G1; @@ -1543,8 +1500,7 @@ static APInt floorOfQuotient(const APInt &A, const APInt &B) { APInt::sdivrem(A, B, Q, R); if (R == 0) return Q; - if ((A.sgt(0) && B.sgt(0)) || - (A.slt(0) && B.slt(0))) + if ((A.sgt(0) && B.sgt(0)) || (A.slt(0) && B.slt(0))) return Q; else return Q - 1; @@ -1556,8 +1512,7 @@ static APInt ceilingOfQuotient(const APInt &A, const APInt &B) { APInt::sdivrem(A, B, Q, R); if (R == 0) return Q; - if ((A.sgt(0) && B.sgt(0)) || - (A.slt(0) && B.slt(0))) + if ((A.sgt(0) && B.sgt(0)) || (A.slt(0) && B.slt(0))) return Q + 1; else return Q; @@ -1733,17 +1688,14 @@ bool DependenceInfo::exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff, return Result.DV[Level].Direction == Dependence::DVEntry::NONE; } - // Return true if the divisor evenly divides the dividend. -static -bool isRemainderZero(const SCEVConstant *Dividend, - const SCEVConstant *Divisor) { +static bool isRemainderZero(const SCEVConstant *Dividend, + const SCEVConstant *Divisor) { const APInt &ConstDividend = Dividend->getAPInt(); const APInt &ConstDivisor = Divisor->getAPInt(); return ConstDividend.srem(ConstDivisor) == 0; } - // weakZeroSrcSIVtest - // From the paper, Practical Dependence Testing, Section 4.2.2 // @@ -1807,11 +1759,11 @@ bool DependenceInfo::weakZeroSrcSIVtest(const SCEV *DstCoeff, const SCEVConstant *ConstCoeff = dyn_cast<SCEVConstant>(DstCoeff); if (!ConstCoeff) return false; - const SCEV *AbsCoeff = - SE->isKnownNegative(ConstCoeff) ? - SE->getNegativeSCEV(ConstCoeff) : ConstCoeff; + const SCEV *AbsCoeff = SE->isKnownNegative(ConstCoeff) + ? SE->getNegativeSCEV(ConstCoeff) + : ConstCoeff; const SCEV *NewDelta = - SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta; + SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta; // check that Delta/SrcCoeff < iteration count // really check NewDelta < count*AbsCoeff @@ -1853,7 +1805,6 @@ bool DependenceInfo::weakZeroSrcSIVtest(const SCEV *DstCoeff, return false; } - // weakZeroDstSIVtest - // From the paper, Practical Dependence Testing, Section 4.2.2 // @@ -1916,11 +1867,11 @@ bool DependenceInfo::weakZeroDstSIVtest(const SCEV *SrcCoeff, const SCEVConstant *ConstCoeff = dyn_cast<SCEVConstant>(SrcCoeff); if (!ConstCoeff) return false; - const SCEV *AbsCoeff = - SE->isKnownNegative(ConstCoeff) ? - SE->getNegativeSCEV(ConstCoeff) : ConstCoeff; + const SCEV *AbsCoeff = SE->isKnownNegative(ConstCoeff) + ? SE->getNegativeSCEV(ConstCoeff) + : ConstCoeff; const SCEV *NewDelta = - SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta; + SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta; // check that Delta/SrcCoeff < iteration count // really check NewDelta < count*AbsCoeff @@ -1962,7 +1913,6 @@ bool DependenceInfo::weakZeroDstSIVtest(const SCEV *SrcCoeff, return false; } - // exactRDIVtest - Tests the RDIV subscript pair for dependence. // Things of the form [c1 + a*i] and [c2 + b*j], // where i and j are induction variable, c1 and c2 are loop invariant, @@ -2084,7 +2034,6 @@ bool DependenceInfo::exactRDIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff, return TL.sgt(TU); } - // symbolicRDIVtest - // In Section 4.5 of the Practical Dependence Testing paper,the authors // introduce a special case of Banerjee's Inequalities (also called the @@ -2167,8 +2116,7 @@ bool DependenceInfo::symbolicRDIVtest(const SCEV *A1, const SCEV *A2, return true; } } - } - else if (SE->isKnownNonPositive(A2)) { + } else if (SE->isKnownNonPositive(A2)) { // a1 >= 0 && a2 <= 0 if (N1 && N2) { // make sure that c2 - c1 <= a1*N1 - a2*N2 @@ -2187,8 +2135,7 @@ bool DependenceInfo::symbolicRDIVtest(const SCEV *A1, const SCEV *A2, return true; } } - } - else if (SE->isKnownNonPositive(A1)) { + } else if (SE->isKnownNonPositive(A1)) { if (SE->isKnownNonNegative(A2)) { // a1 <= 0 && a2 >= 0 if (N1 && N2) { @@ -2207,8 +2154,7 @@ bool DependenceInfo::symbolicRDIVtest(const SCEV *A1, const SCEV *A2, ++SymbolicRDIVindependence; return true; } - } - else if (SE->isKnownNonPositive(A2)) { + } else if (SE->isKnownNonPositive(A2)) { // a1 <= 0 && a2 <= 0 if (N1) { // make sure that a1*N1 <= c2 - c1 @@ -2233,7 +2179,6 @@ bool DependenceInfo::symbolicRDIVtest(const SCEV *A1, const SCEV *A2, return false; } - // testSIV - // When we have a pair of subscripts of the form [c1 + a1*i] and [c2 - a2*i] // where i is an induction variable, c1 and c2 are loop invariant, and a1 and @@ -2260,17 +2205,17 @@ bool DependenceInfo::testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level, Level = mapSrcLoop(CurLoop); bool disproven; if (SrcCoeff == DstCoeff) - disproven = strongSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop, - Level, Result, NewConstraint); + disproven = strongSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop, Level, + Result, NewConstraint); else if (SrcCoeff == SE->getNegativeSCEV(DstCoeff)) disproven = weakCrossingSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop, Level, Result, NewConstraint, SplitIter); else disproven = exactSIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop, Level, Result, NewConstraint); - return disproven || - gcdMIVtest(Src, Dst, Result) || - symbolicRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop, CurLoop); + return disproven || gcdMIVtest(Src, Dst, Result) || + symbolicRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop, + CurLoop); } if (SrcAddRec) { const SCEV *SrcConst = SrcAddRec->getStart(); @@ -2278,9 +2223,9 @@ bool DependenceInfo::testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level, const SCEV *DstConst = Dst; const Loop *CurLoop = SrcAddRec->getLoop(); Level = mapSrcLoop(CurLoop); - return weakZeroDstSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop, - Level, Result, NewConstraint) || - gcdMIVtest(Src, Dst, Result); + return weakZeroDstSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop, Level, + Result, NewConstraint) || + gcdMIVtest(Src, Dst, Result); } if (DstAddRec) { const SCEV *DstConst = DstAddRec->getStart(); @@ -2288,15 +2233,14 @@ bool DependenceInfo::testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level, const SCEV *SrcConst = Src; const Loop *CurLoop = DstAddRec->getLoop(); Level = mapDstLoop(CurLoop); - return weakZeroSrcSIVtest(DstCoeff, SrcConst, DstConst, - CurLoop, Level, Result, NewConstraint) || - gcdMIVtest(Src, Dst, Result); + return weakZeroSrcSIVtest(DstCoeff, SrcConst, DstConst, CurLoop, Level, + Result, NewConstraint) || + gcdMIVtest(Src, Dst, Result); } llvm_unreachable("SIV test expected at least one AddRec"); return false; } - // testRDIV - // When we have a pair of subscripts of the form [c1 + a1*i] and [c2 + a2*j] // where i and j are induction variables, c1 and c2 are loop invariant, @@ -2333,46 +2277,37 @@ bool DependenceInfo::testRDIV(const SCEV *Src, const SCEV *Dst, DstConst = DstAddRec->getStart(); DstCoeff = DstAddRec->getStepRecurrence(*SE); DstLoop = DstAddRec->getLoop(); - } - else if (SrcAddRec) { + } else if (SrcAddRec) { if (const SCEVAddRecExpr *tmpAddRec = - dyn_cast<SCEVAddRecExpr>(SrcAddRec->getStart())) { + dyn_cast<SCEVAddRecExpr>(SrcAddRec->getStart())) { SrcConst = tmpAddRec->getStart(); SrcCoeff = tmpAddRec->getStepRecurrence(*SE); SrcLoop = tmpAddRec->getLoop(); DstConst = Dst; DstCoeff = SE->getNegativeSCEV(SrcAddRec->getStepRecurrence(*SE)); DstLoop = SrcAddRec->getLoop(); - } - else + } else llvm_unreachable("RDIV reached by surprising SCEVs"); - } - else if (DstAddRec) { + } else if (DstAddRec) { if (const SCEVAddRecExpr *tmpAddRec = - dyn_cast<SCEVAddRecExpr>(DstAddRec->getStart())) { + dyn_cast<SCEVAddRecExpr>(DstAddRec->getStart())) { DstConst = tmpAddRec->getStart(); DstCoeff = tmpAddRec->getStepRecurrence(*SE); DstLoop = tmpAddRec->getLoop(); SrcConst = Src; SrcCoeff = SE->getNegativeSCEV(DstAddRec->getStepRecurrence(*SE)); SrcLoop = DstAddRec->getLoop(); - } - else + } else llvm_unreachable("RDIV reached by surprising SCEVs"); - } - else + } else llvm_unreachable("RDIV expected at least one AddRec"); - return exactRDIVtest(SrcCoeff, DstCoeff, - SrcConst, DstConst, - SrcLoop, DstLoop, + return exactRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, SrcLoop, DstLoop, Result) || - gcdMIVtest(Src, Dst, Result) || - symbolicRDIVtest(SrcCoeff, DstCoeff, - SrcConst, DstConst, - SrcLoop, DstLoop); + gcdMIVtest(Src, Dst, Result) || + symbolicRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, SrcLoop, + DstLoop); } - // Tests the single-subscript MIV pair (Src and Dst) for dependence. // Return true if dependence disproved. // Can sometimes refine direction vectors. @@ -2383,7 +2318,7 @@ bool DependenceInfo::testMIV(const SCEV *Src, const SCEV *Dst, LLVM_DEBUG(dbgs() << " dst = " << *Dst << "\n"); Result.Consistent = false; return gcdMIVtest(Src, Dst, Result) || - banerjeeMIVtest(Src, Dst, Loops, Result); + banerjeeMIVtest(Src, Dst, Loops, Result); } // Given a product, e.g., 10*X*Y, returns the first constant operand, @@ -2428,7 +2363,7 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst, // we can't quit the loop just because the GCD == 1. const SCEV *Coefficients = Src; while (const SCEVAddRecExpr *AddRec = - dyn_cast<SCEVAddRecExpr>(Coefficients)) { + dyn_cast<SCEVAddRecExpr>(Coefficients)) { const SCEV *Coeff = AddRec->getStepRecurrence(*SE); // If the coefficient is the product of a constant and other stuff, // we can use the constant in the GCD computation. @@ -2446,7 +2381,7 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst, // we can't quit the loop just because the GCD == 1. Coefficients = Dst; while (const SCEVAddRecExpr *AddRec = - dyn_cast<SCEVAddRecExpr>(Coefficients)) { + dyn_cast<SCEVAddRecExpr>(Coefficients)) { const SCEV *Coeff = AddRec->getStepRecurrence(*SE); // If the coefficient is the product of a constant and other stuff, // we can use the constant in the GCD computation. @@ -2468,16 +2403,14 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst, if (isa<SCEVConstant>(Operand)) { assert(!Constant && "Surprised to find multiple constants"); Constant = cast<SCEVConstant>(Operand); - } - else if (const SCEVMulExpr *Product = dyn_cast<SCEVMulExpr>(Operand)) { + } else if (const SCEVMulExpr *Product = dyn_cast<SCEVMulExpr>(Operand)) { // Search for constant operand to participate in GCD; // If none found; return false. std::optional<APInt> ConstOp = getConstantPart(Product); if (!ConstOp) return false; ExtraGCD = APIntOps::GreatestCommonDivisor(ExtraGCD, ConstOp->abs()); - } - else + } else return false; } } @@ -2512,7 +2445,7 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst, bool Improved = false; Coefficients = Src; while (const SCEVAddRecExpr *AddRec = - dyn_cast<SCEVAddRecExpr>(Coefficients)) { + dyn_cast<SCEVAddRecExpr>(Coefficients)) { Coefficients = AddRec->getStart(); const Loop *CurLoop = AddRec->getLoop(); RunningGCD = ExtraGCD; @@ -2578,7 +2511,6 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst, return false; } - //===----------------------------------------------------------------------===// // banerjeeMIVtest - // Use Banerjee's Inequalities to test an MIV subscript pair. @@ -2652,8 +2584,8 @@ bool DependenceInfo::banerjeeMIVtest(const SCEV *Src, const SCEV *Dst, if (testBounds(Dependence::DVEntry::ALL, 0, Bound, Delta)) { // Explore the direction vector hierarchy. unsigned DepthExpanded = 0; - unsigned NewDeps = exploreDirections(1, A, B, Bound, - Loops, DepthExpanded, Delta); + unsigned NewDeps = + exploreDirections(1, A, B, Bound, Loops, DepthExpanded, Delta); if (NewDeps > 0) { bool Improved = false; for (unsigned K = 1; K <= CommonLevels; ++K) { @@ -2670,23 +2602,20 @@ bool DependenceInfo::banerjeeMIVtest(const SCEV *Src, const SCEV *Dst, } if (Improved) ++BanerjeeSuccesses; - } - else { + } else { ++BanerjeeIndependence; Disproved = true; } - } - else { + } else { ++BanerjeeIndependence; Disproved = true; } - delete [] Bound; - delete [] A; - delete [] B; + delete[] Bound; + delete[] A; + delete[] B; return Disproved; } - // Hierarchically expands the direction vector // search space, combining the directions of discovered dependences // in the DirSet field of Bound. Returns the number of distinct @@ -2788,27 +2717,26 @@ unsigned DependenceInfo::exploreDirections(unsigned Level, CoefficientInfo *A, // test bounds for <, *, *, ... if (testBounds(Dependence::DVEntry::LT, Level, Bound, Delta)) - NewDeps += exploreDirections(Level + 1, A, B, Bound, - Loops, DepthExpanded, Delta); + NewDeps += exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded, + Delta); // Test bounds for =, *, *, ... if (testBounds(Dependence::DVEntry::EQ, Level, Bound, Delta)) - NewDeps += exploreDirections(Level + 1, A, B, Bound, - Loops, DepthExpanded, Delta); + NewDeps += exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded, + Delta); // test bounds for >, *, *, ... if (testBounds(Dependence::DVEntry::GT, Level, Bound, Delta)) - NewDeps += exploreDirections(Level + 1, A, B, Bound, - Loops, DepthExpanded, Delta); + NewDeps += exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded, + Delta); Bound[Level].Direction = Dependence::DVEntry::ALL; return NewDeps; - } - else - return exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded, Delta); + } else + return exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded, + Delta); } - // Returns true iff the current bounds are plausible. bool DependenceInfo::testBounds(unsigned char DirKind, unsigned Level, BoundInfo *Bound, const SCEV *Delta) const { @@ -2822,7 +2750,6 @@ bool DependenceInfo::testBounds(unsigned char DirKind, unsigned Level, return true; } - // Computes the upper and lower bounds for level K // using the * direction. Records them in Bound. // Wolfe gives the equations @@ -2840,17 +2767,16 @@ bool DependenceInfo::testBounds(unsigned char DirKind, unsigned Level, // and the upper bound is always >= 0. void DependenceInfo::findBoundsALL(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, unsigned K) const { - Bound[K].Lower[Dependence::DVEntry::ALL] = nullptr; // Default value = -infinity. - Bound[K].Upper[Dependence::DVEntry::ALL] = nullptr; // Default value = +infinity. + Bound[K].Lower[Dependence::DVEntry::ALL] = + nullptr; // Default value = -infinity. + Bound[K].Upper[Dependence::DVEntry::ALL] = + nullptr; // Default value = +infinity. if (Bound[K].Iterations) { - Bound[K].Lower[Dependence::DVEntry::ALL] = - SE->getMulExpr(SE->getMinusSCEV(A[K].NegPart, B[K].PosPart), - Bound[K].Iterations); - Bound[K].Upper[Dependence::DVEntry::ALL] = - SE->getMulExpr(SE->getMinusSCEV(A[K].PosPart, B[K].NegPart), - Bound[K].Iterations); - } - else { + Bound[K].Lower[Dependence::DVEntry::ALL] = SE->getMulExpr( + SE->getMinusSCEV(A[K].NegPart, B[K].PosPart), Bound[K].Iterations); + Bound[K].Upper[Dependence::DVEntry::ALL] = SE->getMulExpr( + SE->getMinusSCEV(A[K].PosPart, B[K].NegPart), Bound[K].Iterations); + } else { // If the difference is 0, we won't need to know the number of iterations. if (isKnownPredicate(CmpInst::ICMP_EQ, A[K].NegPart, B[K].PosPart)) Bound[K].Lower[Dependence::DVEntry::ALL] = @@ -2861,7 +2787,6 @@ void DependenceInfo::findBoundsALL(CoefficientInfo *A, CoefficientInfo *B, } } - // Computes the upper and lower bounds for level K // using the = direction. Records them in Bound. // Wolfe gives the equations @@ -2879,18 +2804,19 @@ void DependenceInfo::findBoundsALL(CoefficientInfo *A, CoefficientInfo *B, // and the upper bound is always >= 0. void DependenceInfo::findBoundsEQ(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, unsigned K) const { - Bound[K].Lower[Dependence::DVEntry::EQ] = nullptr; // Default value = -infinity. - Bound[K].Upper[Dependence::DVEntry::EQ] = nullptr; // Default value = +infinity. + Bound[K].Lower[Dependence::DVEntry::EQ] = + nullptr; // Default value = -infinity. + Bound[K].Upper[Dependence::DVEntry::EQ] = + nullptr; // Default value = +infinity. if (Bound[K].Iterations) { const SCEV *Delta = SE->getMinusSCEV(A[K].Coeff, B[K].Coeff); const SCEV *NegativePart = getNegativePart(Delta); Bound[K].Lower[Dependence::DVEntry::EQ] = - SE->getMulExpr(NegativePart, Bound[K].Iterations); + SE->getMulExpr(NegativePart, Bound[K].Iterations); const SCEV *PositivePart = getPositivePart(Delta); Bound[K].Upper[Dependence::DVEntry::EQ] = - SE->getMulExpr(PositivePart, Bound[K].Iterations); - } - else { + SE->getMulExpr(PositivePart, Bound[K].Iterations); + } else { // If the positive/negative part of the difference is 0, // we won't need to know the number of iterations. const SCEV *Delta = SE->getMinusSCEV(A[K].Coeff, B[K].Coeff); @@ -2903,7 +2829,6 @@ void DependenceInfo::findBoundsEQ(CoefficientInfo *A, CoefficientInfo *B, } } - // Computes the upper and lower bounds for level K // using the < direction. Records them in Bound. // Wolfe gives the equations @@ -2919,35 +2844,35 @@ void DependenceInfo::findBoundsEQ(CoefficientInfo *A, CoefficientInfo *B, // We must be careful to handle the case where the upper bound is unknown. void DependenceInfo::findBoundsLT(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, unsigned K) const { - Bound[K].Lower[Dependence::DVEntry::LT] = nullptr; // Default value = -infinity. - Bound[K].Upper[Dependence::DVEntry::LT] = nullptr; // Default value = +infinity. + Bound[K].Lower[Dependence::DVEntry::LT] = + nullptr; // Default value = -infinity. + Bound[K].Upper[Dependence::DVEntry::LT] = + nullptr; // Default value = +infinity. if (Bound[K].Iterations) { const SCEV *Iter_1 = SE->getMinusSCEV( Bound[K].Iterations, SE->getOne(Bound[K].Iterations->getType())); const SCEV *NegPart = - getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff)); + getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff)); Bound[K].Lower[Dependence::DVEntry::LT] = - SE->getMinusSCEV(SE->getMulExpr(NegPart, Iter_1), B[K].Coeff); + SE->getMinusSCEV(SE->getMulExpr(NegPart, Iter_1), B[K].Coeff); const SCEV *PosPart = - getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff)); + getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff)); Bound[K].Upper[Dependence::DVEntry::LT] = - SE->getMinusSCEV(SE->getMulExpr(PosPart, Iter_1), B[K].Coeff); - } - else { + SE->getMinusSCEV(SE->getMulExpr(PosPart, Iter_1), B[K].Coeff); + } else { // If the positive/negative part of the difference is 0, // we won't need to know the number of iterations. const SCEV *NegPart = - getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff)); + getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff)); if (NegPart->isZero()) Bound[K].Lower[Dependence::DVEntry::LT] = SE->getNegativeSCEV(B[K].Coeff); const SCEV *PosPart = - getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff)); + getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff)); if (PosPart->isZero()) Bound[K].Upper[Dependence::DVEntry::LT] = SE->getNegativeSCEV(B[K].Coeff); } } - // Computes the upper and lower bounds for level K // using the > direction. Records them in Bound. // Wolfe gives the equations @@ -2963,45 +2888,45 @@ void DependenceInfo::findBoundsLT(CoefficientInfo *A, CoefficientInfo *B, // We must be careful to handle the case where the upper bound is unknown. void DependenceInfo::findBoundsGT(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, unsigned K) const { - Bound[K].Lower[Dependence::DVEntry::GT] = nullptr; // Default value = -infinity. - Bound[K].Upper[Dependence::DVEntry::GT] = nullptr; // Default value = +infinity. + Bound[K].Lower[Dependence::DVEntry::GT] = + nullptr; // Default value = -infinity. + Bound[K].Upper[Dependence::DVEntry::GT] = + nullptr; // Default value = +infinity. if (Bound[K].Iterations) { const SCEV *Iter_1 = SE->getMinusSCEV( Bound[K].Iterations, SE->getOne(Bound[K].Iterations->getType())); const SCEV *NegPart = - getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart)); + getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart)); Bound[K].Lower[Dependence::DVEntry::GT] = - SE->getAddExpr(SE->getMulExpr(NegPart, Iter_1), A[K].Coeff); + SE->getAddExpr(SE->getMulExpr(NegPart, Iter_1), A[K].Coeff); const SCEV *PosPart = - getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart)); + getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart)); Bound[K].Upper[Dependence::DVEntry::GT] = - SE->getAddExpr(SE->getMulExpr(PosPart, Iter_1), A[K].Coeff); - } - else { + SE->getAddExpr(SE->getMulExpr(PosPart, Iter_1), A[K].Coeff); + } else { // If the positive/negative part of the difference is 0, // we won't need to know the number of iterations. - const SCEV *NegPart = getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart)); + const SCEV *NegPart = + getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart)); if (NegPart->isZero()) Bound[K].Lower[Dependence::DVEntry::GT] = A[K].Coeff; - const SCEV *PosPart = getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart)); + const SCEV *PosPart = + getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart)); if (PosPart->isZero()) Bound[K].Upper[Dependence::DVEntry::GT] = A[K].Coeff; } } - // X^+ = max(X, 0) const SCEV *DependenceInfo::getPositivePart(const SCEV *X) const { return SE->getSMaxExpr(X, SE->getZero(X->getType())); } - // X^- = min(X, 0) const SCEV *DependenceInfo::getNegativePart(const SCEV *X) const { return SE->getSMinExpr(X, SE->getZero(X->getType())); } - // Walks through the subscript, // collecting each coefficient, the associated loop bounds, // and recording its positive and negative parts for later use. @@ -3046,7 +2971,6 @@ DependenceInfo::collectCoeffInfo(const SCEV *Subscript, bool SrcFlag, return CI; } - // Looks through all the bounds info and // computes the lower bound given the current direction settings // at each level. If the lower bound for any level is -inf, @@ -3062,7 +2986,6 @@ const SCEV *DependenceInfo::getLowerBound(BoundInfo *Bound) const { return Sum; } - // Looks through all the bounds info and // computes the upper bound given the current direction settings // at each level. If the upper bound at any level is +inf, @@ -3078,7 +3001,6 @@ const SCEV *DependenceInfo::getUpperBound(BoundInfo *Bound) const { return Sum; } - //===----------------------------------------------------------------------===// // Constraint manipulation for Delta test. @@ -3098,7 +3020,6 @@ const SCEV *DependenceInfo::findCoefficient(const SCEV *Expr, return findCoefficient(AddRec->getStart(), TargetLoop); } - // Given a linear SCEV, // return the SCEV given by zeroing out the coefficient // corresponding to the specified loop. @@ -3112,12 +3033,10 @@ const SCEV *DependenceInfo::zeroCoefficient(const SCEV *Expr, if (AddRec->getLoop() == TargetLoop) return AddRec->getStart(); return SE->getAddRecExpr(zeroCoefficient(AddRec->getStart(), TargetLoop), - AddRec->getStepRecurrence(*SE), - AddRec->getLoop(), + AddRec->getStepRecurrence(*SE), AddRec->getLoop(), AddRec->getNoWrapFlags()); } - // Given a linear SCEV Expr, // return the SCEV given by adding some Value to the // coefficient corresponding to the specified TargetLoop. @@ -3128,17 +3047,13 @@ const SCEV *DependenceInfo::addToCoefficient(const SCEV *Expr, const SCEV *Value) const { const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Expr); if (!AddRec) // create a new addRec - return SE->getAddRecExpr(Expr, - Value, - TargetLoop, + return SE->getAddRecExpr(Expr, Value, TargetLoop, SCEV::FlagAnyWrap); // Worst case, with no info. if (AddRec->getLoop() == TargetLoop) { const SCEV *Sum = SE->getAddExpr(AddRec->getStepRecurrence(*SE), Value); if (Sum->isZero()) return AddRec->getStart(); - return SE->getAddRecExpr(AddRec->getStart(), - Sum, - AddRec->getLoop(), + return SE->getAddRecExpr(AddRec->getStart(), Sum, AddRec->getLoop(), AddRec->getNoWrapFlags()); } if (SE->isLoopInvariant(AddRec, TargetLoop)) @@ -3149,7 +3064,6 @@ const SCEV *DependenceInfo::addToCoefficient(const SCEV *Expr, AddRec->getNoWrapFlags()); } - // Review the constraints, looking for opportunities // to simplify a subscript pair (Src and Dst). // Return true if some simplification occurs. @@ -3178,7 +3092,6 @@ bool DependenceInfo::propagate(const SCEV *&Src, const SCEV *&Dst, return Result; } - // Attempt to propagate a distance // constraint into a subscript pair (Src and Dst). // Return true if some simplification occurs. @@ -3204,7 +3117,6 @@ bool DependenceInfo::propagateDistance(const SCEV *&Src, const SCEV *&Dst, return true; } - // Attempt to propagate a line // constraint into a subscript pair (Src and Dst). // Return true if some simplification occurs. @@ -3224,22 +3136,22 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst, if (A->isZero()) { const SCEVConstant *Bconst = dyn_cast<SCEVConstant>(B); const SCEVConstant *Cconst = dyn_cast<SCEVConstant>(C); - if (!Bconst || !Cconst) return false; + if (!Bconst || !Cconst) + return false; APInt Beta = Bconst->getAPInt(); APInt Charlie = Cconst->getAPInt(); APInt CdivB = Charlie.sdiv(Beta); assert(Charlie.srem(Beta) == 0 && "C should be evenly divisible by B"); const SCEV *AP_K = findCoefficient(Dst, CurLoop); - // Src = SE->getAddExpr(Src, SE->getMulExpr(AP_K, SE->getConstant(CdivB))); Src = SE->getMinusSCEV(Src, SE->getMulExpr(AP_K, SE->getConstant(CdivB))); Dst = zeroCoefficient(Dst, CurLoop); if (!findCoefficient(Src, CurLoop)->isZero()) Consistent = false; - } - else if (B->isZero()) { + } else if (B->isZero()) { const SCEVConstant *Aconst = dyn_cast<SCEVConstant>(A); const SCEVConstant *Cconst = dyn_cast<SCEVConstant>(C); - if (!Aconst || !Cconst) return false; + if (!Aconst || !Cconst) + return false; APInt Alpha = Aconst->getAPInt(); APInt Charlie = Cconst->getAPInt(); APInt CdivA = Charlie.sdiv(Alpha); @@ -3249,11 +3161,11 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst, Src = zeroCoefficient(Src, CurLoop); if (!findCoefficient(Dst, CurLoop)->isZero()) Consistent = false; - } - else if (isKnownPredicate(CmpInst::ICMP_EQ, A, B)) { + } else if (isKnownPredicate(CmpInst::ICMP_EQ, A, B)) { const SCEVConstant *Aconst = dyn_cast<SCEVConstant>(A); const SCEVConstant *Cconst = dyn_cast<SCEVConstant>(C); - if (!Aconst || !Cconst) return false; + if (!Aconst || !Cconst) + return false; APInt Alpha = Aconst->getAPInt(); APInt Charlie = Cconst->getAPInt(); APInt CdivA = Charlie.sdiv(Alpha); @@ -3264,8 +3176,7 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst, Dst = addToCoefficient(Dst, CurLoop, A_K); if (!findCoefficient(Dst, CurLoop)->isZero()) Consistent = false; - } - else { + } else { // paper is incorrect here, or perhaps just misleading const SCEV *A_K = findCoefficient(Src, CurLoop); Src = SE->getMulExpr(Src, A); @@ -3281,7 +3192,6 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst, return true; } - // Attempt to propagate a point // constraint into a subscript pair (Src and Dst). // Return true if some simplification occurs. @@ -3302,7 +3212,6 @@ bool DependenceInfo::propagatePoint(const SCEV *&Src, const SCEV *&Dst, return true; } - // Update direction vector entry based on the current constraint. void DependenceInfo::updateDirection(Dependence::DVEntry &Level, const Constraint &CurConstraint) const { @@ -3322,34 +3231,28 @@ void DependenceInfo::updateDirection(Dependence::DVEntry &Level, if (!SE->isKnownNonNegative(Level.Distance)) // if may be negative NewDirection |= Dependence::DVEntry::GT; Level.Direction &= NewDirection; - } - else if (CurConstraint.isLine()) { + } else if (CurConstraint.isLine()) { Level.Scalar = false; Level.Distance = nullptr; // direction should be accurate - } - else if (CurConstraint.isPoint()) { + } else if (CurConstraint.isPoint()) { Level.Scalar = false; Level.Distance = nullptr; unsigned NewDirection = Dependence::DVEntry::NONE; - if (!isKnownPredicate(CmpInst::ICMP_NE, - CurConstraint.getY(), + if (!isKnownPredicate(CmpInst::ICMP_NE, CurConstraint.getY(), CurConstraint.getX())) // if X may be = Y NewDirection |= Dependence::DVEntry::EQ; - if (!isKnownPredicate(CmpInst::ICMP_SLE, - CurConstraint.getY(), + if (!isKnownPredicate(CmpInst::ICMP_SLE, CurConstraint.getY(), CurConstraint.getX())) // if Y may be > X NewDirection |= Dependence::DVEntry::LT; - if (!isKnownPredicate(CmpInst::ICMP_SGE, - CurConstraint.getY(), + if (!isKnownPredicate(CmpInst::ICMP_SGE, CurConstraint.getY(), CurConstraint.getX())) // if Y may be < X NewDirection |= Dependence::DVEntry::GT; Level.Direction &= NewDirection; - } - else + } else llvm_unreachable("constraint has unexpected kind"); } @@ -3425,7 +3328,7 @@ bool DependenceInfo::tryDelinearizeFixedSize( dyn_cast<SCEVUnknown>(SE->getPointerBase(DstAccessFn)); assert(SrcBase && DstBase && SrcBase == DstBase && "expected src and dst scev unknowns to be equal"); - }); + }); SmallVector<int, 4> SrcSizes; SmallVector<int, 4> DstSizes; @@ -3737,9 +3640,8 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, Pair[P].Group.resize(Pairs); removeMatchingExtensions(&Pair[P]); Pair[P].Classification = - classifyPair(Pair[P].Src, LI->getLoopFor(Src->getParent()), - Pair[P].Dst, LI->getLoopFor(Dst->getParent()), - Pair[P].Loops); + classifyPair(Pair[P].Src, LI->getLoopFor(Src->getParent()), Pair[P].Dst, + LI->getLoopFor(Dst->getParent()), Pair[P].Loops); Pair[P].GroupLoops = Pair[P].Loops; Pair[P].Group.set(P); LLVM_DEBUG(dbgs() << " subscript " << P << "\n"); @@ -3814,18 +3716,15 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, if (Pair[SI].Classification == Subscript::NonLinear) { // ignore these, but collect loops for later ++NonlinearSubscriptPairs; - collectCommonLoops(Pair[SI].Src, - LI->getLoopFor(Src->getParent()), + collectCommonLoops(Pair[SI].Src, LI->getLoopFor(Src->getParent()), Pair[SI].Loops); - collectCommonLoops(Pair[SI].Dst, - LI->getLoopFor(Dst->getParent()), + collectCommonLoops(Pair[SI].Dst, LI->getLoopFor(Dst->getParent()), Pair[SI].Loops); Result.Consistent = false; } else if (Pair[SI].Classification == Subscript::ZIV) { // always separable Separable.set(SI); - } - else { + } else { // SIV, RDIV, or MIV, so check for coupled group bool Done = true; for (unsigned SJ = SI + 1; SJ < Pairs; ++SJ) { @@ -3843,8 +3742,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, if (Pair[SI].Group.count() == 1) { Separable.set(SI); ++SeparableSubscriptPairs; - } - else { + } else { Coupled.set(SI); ++CoupledSubscriptPairs; } @@ -3950,10 +3848,9 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, Constraints, Result.Consistent)) { LLVM_DEBUG(dbgs() << "\t Changed\n"); ++DeltaPropagations; - Pair[SJ].Classification = - classifyPair(Pair[SJ].Src, LI->getLoopFor(Src->getParent()), - Pair[SJ].Dst, LI->getLoopFor(Dst->getParent()), - Pair[SJ].Loops); + Pair[SJ].Classification = classifyPair( + Pair[SJ].Src, LI->getLoopFor(Src->getParent()), Pair[SJ].Dst, + LI->getLoopFor(Dst->getParent()), Pair[SJ].Loops); switch (Pair[SJ].Classification) { case Subscript::ZIV: LLVM_DEBUG(dbgs() << "ZIV\n"); @@ -3995,8 +3892,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, LLVM_DEBUG(dbgs() << "MIV test\n"); if (testMIV(Pair[SJ].Src, Pair[SJ].Dst, Pair[SJ].Loops, Result)) return nullptr; - } - else + } else llvm_unreachable("expected only MIV subscripts at this point"); } @@ -4052,8 +3948,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, break; } } - } - else { + } else { // On the other hand, if all directions are equal and there's no // loop-independent dependence possible, then no dependence exists. bool AllEqual = true; @@ -4158,9 +4053,8 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep, Pair[P].Group.resize(Pairs); removeMatchingExtensions(&Pair[P]); Pair[P].Classification = - classifyPair(Pair[P].Src, LI->getLoopFor(Src->getParent()), - Pair[P].Dst, LI->getLoopFor(Dst->getParent()), - Pair[P].Loops); + classifyPair(Pair[P].Src, LI->getLoopFor(Src->getParent()), Pair[P].Dst, + LI->getLoopFor(Dst->getParent()), Pair[P].Loops); Pair[P].GroupLoops = Pair[P].Loops; Pair[P].Group.set(P); } @@ -4172,15 +4066,12 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep, for (unsigned SI = 0; SI < Pairs; ++SI) { if (Pair[SI].Classification == Subscript::NonLinear) { // ignore these, but collect loops for later - collectCommonLoops(Pair[SI].Src, - LI->getLoopFor(Src->getParent()), + collectCommonLoops(Pair[SI].Src, LI->getLoopFor(Src->getParent()), Pair[SI].Loops); - collectCommonLoops(Pair[SI].Dst, - LI->getLoopFor(Dst->getParent()), + collectCommonLoops(Pair[SI].Dst, LI->getLoopFor(Dst->getParent()), Pair[SI].Loops); Result.Consistent = false; - } - else if (Pair[SI].Classification == Subscript::ZIV) + } else if (Pair[SI].Classification == Subscript::ZIV) Separable.set(SI); else { // SIV, RDIV, or MIV, so check for coupled group @@ -4214,8 +4105,8 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep, case Subscript::SIV: { unsigned Level; const SCEV *SplitIter = nullptr; - (void) testSIV(Pair[SI].Src, Pair[SI].Dst, Level, - Result, NewConstraint, SplitIter); + (void)testSIV(Pair[SI].Src, Pair[SI].Dst, Level, Result, NewConstraint, + SplitIter); if (Level == SplitLevel) { assert(SplitIter != nullptr); return SplitIter; diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp index b3b4c37..425ea31 100644 --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -81,6 +81,7 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) { case Intrinsic::exp: case Intrinsic::exp10: case Intrinsic::exp2: + case Intrinsic::ldexp: case Intrinsic::log: case Intrinsic::log10: case Intrinsic::log2: @@ -108,6 +109,8 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) { case Intrinsic::canonicalize: case Intrinsic::fptosi_sat: case Intrinsic::fptoui_sat: + case Intrinsic::lround: + case Intrinsic::llround: case Intrinsic::lrint: case Intrinsic::llrint: case Intrinsic::ucmp: @@ -189,6 +192,8 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg( switch (ID) { case Intrinsic::fptosi_sat: case Intrinsic::fptoui_sat: + case Intrinsic::lround: + case Intrinsic::llround: case Intrinsic::lrint: case Intrinsic::llrint: case Intrinsic::vp_lrint: @@ -203,6 +208,7 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg( case Intrinsic::vp_is_fpclass: return OpdIdx == 0; case Intrinsic::powi: + case Intrinsic::ldexp: return OpdIdx == -1 || OpdIdx == 1; default: return OpdIdx == -1; diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp index 9ba1782..0f3ec8b 100644 --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -132,9 +132,10 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, unsigned i = 0; unsigned NumFixedArgs = CB.getFunctionType()->getNumParams(); for (const auto &Arg : CB.args()) { - ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i), - i < NumFixedArgs}; + ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i)}; setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB); + if (i >= NumFixedArgs) + OrigArg.Flags[0].setVarArg(); // If we have an explicit sret argument that is an Instruction, (i.e., it // might point to function-local memory), we can't meaningfully tail-call. @@ -301,7 +302,7 @@ void CallLowering::splitToValueTypes(const ArgInfo &OrigArg, // double] -> double). SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx), OrigArg.OrigArgIndex, OrigArg.Flags[0], - OrigArg.IsFixed, OrigArg.OrigValue); + OrigArg.OrigValue); return; } @@ -313,7 +314,7 @@ void CallLowering::splitToValueTypes(const ArgInfo &OrigArg, for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) { Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx); SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.OrigArgIndex, - OrigArg.Flags[0], OrigArg.IsFixed); + OrigArg.Flags[0]); if (NeedsRegBlock) SplitArgs.back().Flags[0].setInConsecutiveRegs(); } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 649a310..bfa72bf 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -5630,6 +5630,7 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, case ISD::FDIV: case ISD::FREM: case ISD::FCOPYSIGN: + case ISD::FP_EXTEND: // No poison except from flags (which is handled above) return false; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index d0815e9..868e2f4 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2273,9 +2273,8 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) { Flags.setNoExt(); for (unsigned i = 0; i < NumParts; ++i) { - Outs.push_back(ISD::OutputArg(Flags, - Parts[i].getValueType().getSimpleVT(), - VT, /*isfixed=*/true, 0, 0)); + Outs.push_back(ISD::OutputArg( + Flags, Parts[i].getValueType().getSimpleVT(), VT, 0, 0)); OutVals.push_back(Parts[i]); } } @@ -2291,9 +2290,9 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) { assert(SwiftError.getFunctionArg() && "Need a swift error argument"); ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); Flags.setSwiftError(); - Outs.push_back(ISD::OutputArg( - Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)), - /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0)); + Outs.push_back(ISD::OutputArg(Flags, /*vt=*/TLI.getPointerTy(DL), + /*argvt=*/EVT(TLI.getPointerTy(DL)), + /*origidx=*/1, /*partOffs=*/0)); // Create SDNode for the swifterror virtual register. OutVals.push_back( DAG.getRegister(SwiftError.getOrCreateVRegUseAt( @@ -11124,6 +11123,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL)); Flags.setOrigAlign(OriginalAlignment); + if (i >= CLI.NumFixedArgs) + Flags.setVarArg(); if (Args[i].Ty->isPointerTy()) { Flags.setPointer(); Flags.setPointerAddrSpace( @@ -11246,8 +11247,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { // For scalable vectors the scalable part is currently handled // by individual targets, so we just use the known minimum size here. ISD::OutputArg MyFlags( - Flags, Parts[j].getValueType().getSimpleVT(), VT, - i < CLI.NumFixedArgs, i, + Flags, Parts[j].getValueType().getSimpleVT(), VT, i, j * Parts[j].getValueType().getStoreSize().getKnownMinValue()); if (NumParts > 1 && j == 0) MyFlags.Flags.setSplit(); diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp index 9f525ea..17a01f48 100644 --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -1772,7 +1772,7 @@ void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType, Flags.setZExt(); for (unsigned i = 0; i < NumParts; ++i) - Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isfixed=*/true, 0, 0)); + Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, 0, 0)); } } diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 1f5ff4e..a40de86b 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -8537,7 +8537,7 @@ static void analyzeCallOperands(const AArch64TargetLowering &TLI, if (IsCalleeWin64) { UseVarArgCC = true; } else { - UseVarArgCC = !Outs[i].IsFixed; + UseVarArgCC = ArgFlags.isVarArg(); } } @@ -8982,7 +8982,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, unsigned NumArgs = Outs.size(); for (unsigned i = 0; i != NumArgs; ++i) { - if (!Outs[i].IsFixed && Outs[i].VT.isScalableVector()) + if (Outs[i].Flags.isVarArg() && Outs[i].VT.isScalableVector()) report_fatal_error("Passing SVE types to variadic functions is " "currently not supported"); } diff --git a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp index b97d622..fd4ef2a 100644 --- a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp +++ b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp @@ -8,8 +8,8 @@ // // This pass performs below peephole optimizations on MIR level. // -// 1. MOVi32imm + ANDS?Wrr ==> ANDWri + ANDS?Wri -// MOVi64imm + ANDS?Xrr ==> ANDXri + ANDS?Xri +// 1. MOVi32imm + (ANDS?|EOR|ORR)Wrr ==> (AND|EOR|ORR)Wri + (ANDS?|EOR|ORR)Wri +// MOVi64imm + (ANDS?|EOR|ORR)Xrr ==> (AND|EOR|ORR)Xri + (ANDS?|EOR|ORR)Xri // // 2. MOVi32imm + ADDWrr ==> ADDWRi + ADDWRi // MOVi64imm + ADDXrr ==> ADDXri + ADDXri @@ -128,6 +128,7 @@ struct AArch64MIPeepholeOpt : public MachineFunctionPass { // Strategy used to split logical immediate bitmasks. enum class SplitStrategy { Intersect, + Disjoint, }; template <typename T> bool trySplitLogicalImm(unsigned Opc, MachineInstr &MI, @@ -163,6 +164,7 @@ INITIALIZE_PASS(AArch64MIPeepholeOpt, "aarch64-mi-peephole-opt", template <typename T> static bool splitBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc, T &Imm2Enc) { T UImm = static_cast<T>(Imm); + assert(UImm && (UImm != ~static_cast<T>(0)) && "Invalid immediate!"); // The bitmask immediate consists of consecutive ones. Let's say there is // constant 0b00000000001000000000010000000000 which does not consist of @@ -191,18 +193,47 @@ static bool splitBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc, T &Imm2Enc) { } template <typename T> +static bool splitDisjointBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc, + T &Imm2Enc) { + assert(Imm && (Imm != ~static_cast<T>(0)) && "Invalid immediate!"); + + // Try to split a bitmask of the form 0b00000000011000000000011110000000 into + // two disjoint masks such as 0b00000000011000000000000000000000 and + // 0b00000000000000000000011110000000 where the inclusive/exclusive OR of the + // new masks match the original mask. + unsigned LowestBitSet = llvm::countr_zero(Imm); + unsigned LowestGapBitUnset = + LowestBitSet + llvm::countr_one(Imm >> LowestBitSet); + + // Create a mask for the least significant group of consecutive ones. + assert(LowestGapBitUnset < sizeof(T) * CHAR_BIT && "Undefined behaviour!"); + T NewImm1 = (static_cast<T>(1) << LowestGapBitUnset) - + (static_cast<T>(1) << LowestBitSet); + // Create a disjoint mask for the remaining ones. + T NewImm2 = Imm & ~NewImm1; + + // Do not split if NewImm2 is not a valid bitmask immediate. + if (!AArch64_AM::isLogicalImmediate(NewImm2, RegSize)) + return false; + + Imm1Enc = AArch64_AM::encodeLogicalImmediate(NewImm1, RegSize); + Imm2Enc = AArch64_AM::encodeLogicalImmediate(NewImm2, RegSize); + return true; +} + +template <typename T> bool AArch64MIPeepholeOpt::trySplitLogicalImm(unsigned Opc, MachineInstr &MI, SplitStrategy Strategy, unsigned OtherOpc) { - // Try below transformation. + // Try below transformations. // - // MOVi32imm + ANDS?Wrr ==> ANDWri + ANDS?Wri - // MOVi64imm + ANDS?Xrr ==> ANDXri + ANDS?Xri + // MOVi32imm + (ANDS?|EOR|ORR)Wrr ==> (AND|EOR|ORR)Wri + (ANDS?|EOR|ORR)Wri + // MOVi64imm + (ANDS?|EOR|ORR)Xrr ==> (AND|EOR|ORR)Xri + (ANDS?|EOR|ORR)Xri // // The mov pseudo instruction could be expanded to multiple mov instructions // later. Let's try to split the constant operand of mov instruction into two - // bitmask immediates. It makes only two AND instructions instead of multiple - // mov + and instructions. + // bitmask immediates based on the given split strategy. It makes only two + // logical instructions instead of multiple mov + logic instructions. return splitTwoPartImm<T>( MI, @@ -224,6 +255,9 @@ bool AArch64MIPeepholeOpt::trySplitLogicalImm(unsigned Opc, MachineInstr &MI, case SplitStrategy::Intersect: SplitSucc = splitBitmaskImm(Imm, RegSize, Imm0, Imm1); break; + case SplitStrategy::Disjoint: + SplitSucc = splitDisjointBitmaskImm(Imm, RegSize, Imm0, Imm1); + break; } if (SplitSucc) return std::make_pair(Opc, !OtherOpc ? Opc : OtherOpc); @@ -889,6 +923,22 @@ bool AArch64MIPeepholeOpt::runOnMachineFunction(MachineFunction &MF) { Changed |= trySplitLogicalImm<uint64_t>( AArch64::ANDXri, MI, SplitStrategy::Intersect, AArch64::ANDSXri); break; + case AArch64::EORWrr: + Changed |= trySplitLogicalImm<uint32_t>(AArch64::EORWri, MI, + SplitStrategy::Disjoint); + break; + case AArch64::EORXrr: + Changed |= trySplitLogicalImm<uint64_t>(AArch64::EORXri, MI, + SplitStrategy::Disjoint); + break; + case AArch64::ORRWrr: + Changed |= trySplitLogicalImm<uint32_t>(AArch64::ORRWri, MI, + SplitStrategy::Disjoint); + break; + case AArch64::ORRXrr: + Changed |= trySplitLogicalImm<uint64_t>(AArch64::ORRXri, MI, + SplitStrategy::Disjoint); + break; case AArch64::ORRWrs: Changed |= visitORR(MI); break; diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td index adc984a..1bc1d98 100644 --- a/llvm/lib/Target/AArch64/AArch64Processors.td +++ b/llvm/lib/Target/AArch64/AArch64Processors.td @@ -22,7 +22,8 @@ def TuneA320 : SubtargetFeature<"a320", "ARMProcFamily", "CortexA320", FeatureFuseAES, FeatureFuseAdrpAdd, FeaturePostRAScheduler, - FeatureUseWzrToVecMove]>; + FeatureUseWzrToVecMove, + FeatureUseFixedOverScalableIfEqualCost]>; def TuneA53 : SubtargetFeature<"a53", "ARMProcFamily", "CortexA53", "Cortex-A53 ARM processors", [ @@ -45,7 +46,8 @@ def TuneA510 : SubtargetFeature<"a510", "ARMProcFamily", "CortexA510", FeatureFuseAES, FeatureFuseAdrpAdd, FeaturePostRAScheduler, - FeatureUseWzrToVecMove + FeatureUseWzrToVecMove, + FeatureUseFixedOverScalableIfEqualCost ]>; def TuneA520 : SubtargetFeature<"a520", "ARMProcFamily", "CortexA520", @@ -53,7 +55,8 @@ def TuneA520 : SubtargetFeature<"a520", "ARMProcFamily", "CortexA520", FeatureFuseAES, FeatureFuseAdrpAdd, FeaturePostRAScheduler, - FeatureUseWzrToVecMove]>; + FeatureUseWzrToVecMove, + FeatureUseFixedOverScalableIfEqualCost]>; def TuneA520AE : SubtargetFeature<"a520ae", "ARMProcFamily", "CortexA520", "Cortex-A520AE ARM processors", [ @@ -756,7 +759,6 @@ def ProcessorFeatures { FeatureSB, FeaturePAuth, FeatureSSBS, FeatureSVE, FeatureSVE2, FeatureComplxNum, FeatureCRC, FeatureDotProd, FeatureFPARMv8,FeatureFullFP16, FeatureJS, FeatureLSE, - FeatureUseFixedOverScalableIfEqualCost, FeatureRAS, FeatureRCPC, FeatureRDM, FeatureFPAC]; list<SubtargetFeature> A520 = [HasV9_2aOps, FeaturePerfMon, FeatureAM, FeatureMTE, FeatureETE, FeatureSVEBitPerm, @@ -766,7 +768,6 @@ def ProcessorFeatures { FeatureSVE, FeatureSVE2, FeatureBF16, FeatureComplxNum, FeatureCRC, FeatureFPARMv8, FeatureFullFP16, FeatureMatMulInt8, FeatureJS, FeatureNEON, FeatureLSE, FeatureRAS, FeatureRCPC, FeatureRDM, - FeatureUseFixedOverScalableIfEqualCost, FeatureDotProd, FeatureFPAC]; list<SubtargetFeature> A520AE = [HasV9_2aOps, FeaturePerfMon, FeatureAM, FeatureMTE, FeatureETE, FeatureSVEBitPerm, diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp index 010d0aaa..2155ace 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp @@ -125,7 +125,7 @@ struct AArch64OutgoingValueAssigner bool UseVarArgsCCForFixed = IsCalleeWin && State.isVarArg(); bool Res; - if (Info.IsFixed && !UseVarArgsCCForFixed) { + if (!Flags.isVarArg() && !UseVarArgsCCForFixed) { if (!IsReturn) applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT); Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); @@ -361,7 +361,7 @@ struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler { unsigned MaxSize = MemTy.getSizeInBytes() * 8; // For varargs, we always want to extend them to 8 bytes, in which case // we disable setting a max. - if (!Arg.IsFixed) + if (Arg.Flags[0].isVarArg()) MaxSize = 0; Register ValVReg = Arg.Regs[RegIndex]; diff --git a/llvm/lib/Target/AVR/AVRISelLowering.cpp b/llvm/lib/Target/AVR/AVRISelLowering.cpp index 3955f2a..25ad9ec 100644 --- a/llvm/lib/Target/AVR/AVRISelLowering.cpp +++ b/llvm/lib/Target/AVR/AVRISelLowering.cpp @@ -669,7 +669,7 @@ SDValue AVRTargetLowering::getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, default: { // Turn lhs < rhs with lhs constant into rhs >= lhs+1, this allows // us to fold the constant into the cmp instruction. - RHS = DAG.getConstant(C->getSExtValue() + 1, DL, VT); + RHS = DAG.getSignedConstant(C->getSExtValue() + 1, DL, VT); CC = ISD::SETGE; break; } @@ -713,7 +713,10 @@ SDValue AVRTargetLowering::getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, // Turn lhs < rhs with lhs constant into rhs >= lhs+1, this allows us to // fold the constant into the cmp instruction. if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) { - RHS = DAG.getConstant(C->getSExtValue() + 1, DL, VT); + // Doing a "icmp ugt i16 65535, %0" comparison should have been converted + // already to something else. Assert to make sure this assumption holds. + assert((!C->isAllOnes()) && "integer overflow in comparison transform"); + RHS = DAG.getConstant(C->getZExtValue() + 1, DL, VT); CC = ISD::SETUGE; break; } diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index a5bf0e5..6583a0f 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -6729,8 +6729,7 @@ static bool CC_LoongArchAssign2GRLen(unsigned GRLen, CCState &State, static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI, unsigned ValNo, MVT ValVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, - CCState &State, bool IsFixed, bool IsRet, - Type *OrigTy) { + CCState &State, bool IsRet, Type *OrigTy) { unsigned GRLen = DL.getLargestLegalIntTypeSizeInBits(); assert((GRLen == 32 || GRLen == 64) && "Unspport GRLen"); MVT GRLenVT = GRLen == 32 ? MVT::i32 : MVT::i64; @@ -6752,7 +6751,7 @@ static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI, case LoongArchABI::ABI_LP64F: case LoongArchABI::ABI_ILP32D: case LoongArchABI::ABI_LP64D: - UseGPRForFloat = !IsFixed; + UseGPRForFloat = ArgFlags.isVarArg(); break; case LoongArchABI::ABI_ILP32S: case LoongArchABI::ABI_LP64S: @@ -6766,7 +6765,8 @@ static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI, // will not be passed by registers if the original type is larger than // 2*GRLen, so the register alignment rule does not apply. unsigned TwoGRLenInBytes = (2 * GRLen) / 8; - if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoGRLenInBytes && + if (ArgFlags.isVarArg() && + ArgFlags.getNonZeroOrigAlign() == TwoGRLenInBytes && DL.getTypeAllocSize(OrigTy) == TwoGRLenInBytes) { unsigned RegIdx = State.getFirstUnallocated(ArgGPRs); // Skip 'odd' register if necessary. @@ -6916,7 +6916,7 @@ void LoongArchTargetLowering::analyzeInputArgs( LoongArchABI::ABI ABI = MF.getSubtarget<LoongArchSubtarget>().getTargetABI(); if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Ins[i].Flags, - CCInfo, /*IsFixed=*/true, IsRet, ArgTy)) { + CCInfo, IsRet, ArgTy)) { LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type " << ArgVT << '\n'); llvm_unreachable(""); @@ -6934,7 +6934,7 @@ void LoongArchTargetLowering::analyzeOutputArgs( LoongArchABI::ABI ABI = MF.getSubtarget<LoongArchSubtarget>().getTargetABI(); if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Outs[i].Flags, - CCInfo, Outs[i].IsFixed, IsRet, OrigTy)) { + CCInfo, IsRet, OrigTy)) { LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type " << ArgVT << "\n"); llvm_unreachable(""); @@ -7647,8 +7647,7 @@ bool LoongArchTargetLowering::CanLowerReturn( LoongArchABI::ABI ABI = MF.getSubtarget<LoongArchSubtarget>().getTargetABI(); if (CC_LoongArch(MF.getDataLayout(), ABI, i, Outs[i].VT, CCValAssign::Full, - Outs[i].Flags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, - nullptr)) + Outs[i].Flags, CCInfo, /*IsRet=*/true, nullptr)) return false; } return true; diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h index 6b49a98f..f79ba74 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h @@ -330,7 +330,7 @@ private: unsigned ValNo, MVT ValVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, - bool IsFixed, bool IsRet, Type *OrigTy); + bool IsRet, Type *OrigTy); void analyzeInputArgs(MachineFunction &MF, CCState &CCInfo, const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet, diff --git a/llvm/lib/Target/Mips/MipsCCState.cpp b/llvm/lib/Target/Mips/MipsCCState.cpp index 9e8cd2e..13237c5 100644 --- a/llvm/lib/Target/Mips/MipsCCState.cpp +++ b/llvm/lib/Target/Mips/MipsCCState.cpp @@ -128,12 +128,10 @@ void MipsCCState::PreAnalyzeReturnValue(EVT ArgVT) { OriginalRetWasFloatVector.push_back(originalEVTTypeIsVectorFloat(ArgVT)); } -void MipsCCState::PreAnalyzeCallOperand(const Type *ArgTy, bool IsFixed, - const char *Func) { +void MipsCCState::PreAnalyzeCallOperand(const Type *ArgTy, const char *Func) { OriginalArgWasF128.push_back(originalTypeIsF128(ArgTy, Func)); OriginalArgWasFloat.push_back(ArgTy->isFloatingPointTy()); OriginalArgWasFloatVector.push_back(ArgTy->isVectorTy()); - CallOperandIsFixed.push_back(IsFixed); } /// Identify lowered values that originated from f128, float and sret to vXfXX @@ -148,7 +146,6 @@ void MipsCCState::PreAnalyzeCallOperands( OriginalArgWasF128.push_back(originalTypeIsF128(FuncArg.Ty, Func)); OriginalArgWasFloat.push_back(FuncArg.Ty->isFloatingPointTy()); OriginalArgWasFloatVector.push_back(FuncArg.Ty->isVectorTy()); - CallOperandIsFixed.push_back(Outs[i].IsFixed); } } diff --git a/llvm/lib/Target/Mips/MipsCCState.h b/llvm/lib/Target/Mips/MipsCCState.h index 4229da5..30b68e8 100644 --- a/llvm/lib/Target/Mips/MipsCCState.h +++ b/llvm/lib/Target/Mips/MipsCCState.h @@ -36,7 +36,7 @@ public: static bool originalEVTTypeIsVectorFloat(EVT Ty); static bool originalTypeIsVectorFloat(const Type *Ty); - void PreAnalyzeCallOperand(const Type *ArgTy, bool IsFixed, const char *Func); + void PreAnalyzeCallOperand(const Type *ArgTy, const char *Func); void PreAnalyzeFormalArgument(const Type *ArgTy, ISD::ArgFlagsTy Flags); void PreAnalyzeReturnValue(EVT ArgVT); @@ -86,10 +86,6 @@ private: /// vector. SmallVector<bool, 4> OriginalRetWasFloatVector; - /// Records whether the value was a fixed argument. - /// See ISD::OutputArg::IsFixed, - SmallVector<bool, 4> CallOperandIsFixed; - // Used to handle MIPS16-specific calling convention tweaks. // FIXME: This should probably be a fully fledged calling convention. SpecialCallingConvType SpecialCallingConv; @@ -106,7 +102,6 @@ public: OriginalArgWasF128.clear(); OriginalArgWasFloat.clear(); OriginalArgWasFloatVector.clear(); - CallOperandIsFixed.clear(); PreAnalyzeCallOperands(Outs, FuncArgs, Func); } @@ -213,7 +208,6 @@ public: bool WasOriginalRetVectorFloat(unsigned ValNo) const { return OriginalRetWasFloatVector[ValNo]; } - bool IsCallOperandFixed(unsigned ValNo) { return CallOperandIsFixed[ValNo]; } SpecialCallingConvType getSpecialCallingConv() { return SpecialCallingConv; } }; } diff --git a/llvm/lib/Target/Mips/MipsCallLowering.cpp b/llvm/lib/Target/Mips/MipsCallLowering.cpp index 555773a..fa49108 100644 --- a/llvm/lib/Target/Mips/MipsCallLowering.cpp +++ b/llvm/lib/Target/Mips/MipsCallLowering.cpp @@ -47,7 +47,7 @@ struct MipsOutgoingValueAssigner : public CallLowering::OutgoingValueAssigner { if (IsReturn) State.PreAnalyzeReturnValue(EVT::getEVT(Info.Ty)); else - State.PreAnalyzeCallOperand(Info.Ty, Info.IsFixed, Func); + State.PreAnalyzeCallOperand(Info.Ty, Func); return CallLowering::OutgoingValueAssigner::assignArg( ValNo, OrigVT, ValVT, LocVT, LocInfo, Info, Flags, State); diff --git a/llvm/lib/Target/Mips/MipsCallingConv.td b/llvm/lib/Target/Mips/MipsCallingConv.td index 39e184a..0e5c16c 100644 --- a/llvm/lib/Target/Mips/MipsCallingConv.td +++ b/llvm/lib/Target/Mips/MipsCallingConv.td @@ -29,12 +29,6 @@ class CCIfOrigArgWasFloat<CCAction A> class CCIfOrigArgWasF128<CCAction A> : CCIf<"static_cast<MipsCCState *>(&State)->WasOriginalArgF128(ValNo)", A>; -/// Match if this specific argument is a vararg. -/// This is slightly different fro CCIfIsVarArg which matches if any argument is -/// a vararg. -class CCIfArgIsVarArg<CCAction A> - : CCIf<"!static_cast<MipsCCState *>(&State)->IsCallOperandFixed(ValNo)", A>; - /// Match if the return was a floating point vector. class CCIfOrigArgWasNotVectorFloat<CCAction A> : CCIf<"!static_cast<MipsCCState *>(&State)" @@ -344,7 +338,7 @@ def CC_Mips_VarArg : CallingConv<[ ]>; def CC_Mips : CallingConv<[ - CCIfVarArg<CCIfArgIsVarArg<CCDelegateTo<CC_Mips_VarArg>>>, + CCIfVarArg<CCIfArgVarArg<CCDelegateTo<CC_Mips_VarArg>>>, CCDelegateTo<CC_Mips_FixedArg> ]>; diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp index 9003ace..d4f0cc9 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -4046,6 +4046,18 @@ bool NVPTXTargetLowering::getTgtMemIntrinsic( return true; } + case Intrinsic::nvvm_prefetch_tensormap: { + auto &DL = I.getDataLayout(); + Info.opc = ISD::INTRINSIC_VOID; + Info.memVT = getPointerTy(DL); + Info.ptrVal = I.getArgOperand(0); + Info.offset = 0; + Info.flags = + MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable; + Info.align.reset(); + return true; + } + case Intrinsic::nvvm_ldu_global_i: case Intrinsic::nvvm_ldu_global_f: case Intrinsic::nvvm_ldu_global_p: { diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td index d337192..d4a0ca7 100644 --- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td +++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td @@ -39,6 +39,12 @@ def AS_match { code global = [{ return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_GLOBAL); }]; + code const = [{ + return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_CONST); + }]; + code param = [{ + return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_PARAM); + }]; } @@ -950,33 +956,47 @@ foreach dim = 3...5 in { defm TMA_TENSOR_PF_TILE_GATHER4_2D : TMA_TENSOR_PREFETCH_INTR<5, "tile_gather4", [hasTMACTAGroupSupport]>; -//Prefetch and Prefetchu - -let Predicates = [hasPTX<80>, hasSM<90>] in { - class PREFETCH_INTRS<string InstName> : - BasicNVPTXInst<(outs), (ins ADDR:$addr), - InstName, - [(!cast<Intrinsic>(!strconcat("int_nvvm_", - !subst(".", "_", InstName))) addr:$addr)]>; +//Prefetchu and Prefetch - def PREFETCH_L1 : PREFETCH_INTRS<"prefetch.L1">; - def PREFETCH_L2 : PREFETCH_INTRS<"prefetch.L2">; - def PREFETCH_GLOBAL_L1 : PREFETCH_INTRS<"prefetch.global.L1">; - def PREFETCH_LOCAL_L1 : PREFETCH_INTRS<"prefetch.local.L1">; - def PREFETCH_GLOBAL_L2 : PREFETCH_INTRS<"prefetch.global.L2">; - def PREFETCH_LOCAL_L2 : PREFETCH_INTRS<"prefetch.local.L2">; +defvar frag_pat = (int_nvvm_prefetch_tensormap node:$addr); - def PREFETCH_GLOBAL_L2_EVICT_NORMAL : BasicNVPTXInst<(outs), (ins ADDR:$addr), - "prefetch.global.L2::evict_normal", - [(int_nvvm_prefetch_global_L2_evict_normal addr:$addr)]>; +multiclass PREFETCH_TENSORMAP_PATFRAG<string suffix, code predicate> { + def !tolower(suffix) : PatFrag<!setdagop(frag_pat, ops), frag_pat, predicate>; +} - def PREFETCH_GLOBAL_L2_EVICT_LAST : BasicNVPTXInst<(outs), (ins ADDR:$addr), - "prefetch.global.L2::evict_last", - [(int_nvvm_prefetch_global_L2_evict_last addr:$addr)]>; +defm prefetch_tensormap_ : PREFETCH_TENSORMAP_PATFRAG<"CONST", AS_match.const>; +defm prefetch_tensormap_ : PREFETCH_TENSORMAP_PATFRAG<"GENERIC", AS_match.generic>; +defm prefetch_tensormap_ : PREFETCH_TENSORMAP_PATFRAG<"PARAM", AS_match.param>; - def PREFETCHU_L1 : PREFETCH_INTRS<"prefetchu.L1">; +multiclass PREFETCH_TENSORMAP_INST<string addrspace_name, PatFrag pattern_frag> { + def "" : BasicNVPTXInst<(outs), (ins ADDR:$addr), + "prefetch" # addrspace_name # ".tensormap", + [(pattern_frag addr:$addr)]>, + Requires<[hasPTX<80>, hasSM<90>]>; } +defm PREFETCH_CONST_TENSORMAP : PREFETCH_TENSORMAP_INST<".const", prefetch_tensormap_const>; +defm PREFETCH_GENERIC_TENSORMAP : PREFETCH_TENSORMAP_INST<"", prefetch_tensormap_generic>; +defm PREFETCH_PARAM_TENSORMAP : PREFETCH_TENSORMAP_INST<".param", prefetch_tensormap_param>; + +class PREFETCH_INTRS<string InstName, Intrinsic Intr> : + BasicNVPTXInst<(outs), (ins ADDR:$addr), + InstName, + [(Intr addr:$addr)]>, + Requires<[hasPTX<80>, hasSM<90>]>; + +def PREFETCHU_L1 : PREFETCH_INTRS<"prefetchu.L1", int_nvvm_prefetchu_L1>; +def PREFETCH_L1 : PREFETCH_INTRS<"prefetch.L1", int_nvvm_prefetch_L1>; +def PREFETCH_L2 : PREFETCH_INTRS<"prefetch.L2", int_nvvm_prefetch_L2>; +def PREFETCH_GLOBAL_L1 : PREFETCH_INTRS<"prefetch.global.L1", int_nvvm_prefetch_global_L1>; +def PREFETCH_LOCAL_L1 : PREFETCH_INTRS<"prefetch.local.L1", int_nvvm_prefetch_local_L1>; +def PREFETCH_GLOBAL_L2 : PREFETCH_INTRS<"prefetch.global.L2", int_nvvm_prefetch_global_L2>; +def PREFETCH_LOCAL_L2 : PREFETCH_INTRS<"prefetch.local.L2", int_nvvm_prefetch_local_L2>; +def PREFETCH_GLOBAL_L2_EVICT_NORMAL : PREFETCH_INTRS<"prefetch.global.L2::evict_normal", + int_nvvm_prefetch_global_L2_evict_normal>; +def PREFETCH_GLOBAL_L2_EVICT_LAST : PREFETCH_INTRS<"prefetch.global.L2::evict_last", + int_nvvm_prefetch_global_L2_evict_last>; + //Applypriority intrinsics class APPLYPRIORITY_L2_INTRS<string addrspace> : BasicNVPTXInst<(outs), (ins ADDR:$addr, B64:$size), diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp index 3ae2d9d..f4f8961 100644 --- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp @@ -564,7 +564,8 @@ bool NVPTXTTIImpl::collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes, case Intrinsic::nvvm_isspacep_global: case Intrinsic::nvvm_isspacep_local: case Intrinsic::nvvm_isspacep_shared: - case Intrinsic::nvvm_isspacep_shared_cluster: { + case Intrinsic::nvvm_isspacep_shared_cluster: + case Intrinsic::nvvm_prefetch_tensormap: { OpIndexes.push_back(0); return true; } @@ -587,6 +588,11 @@ Value *NVPTXTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, return ConstantInt::get(II->getType(), *R); return nullptr; } + case Intrinsic::nvvm_prefetch_tensormap: { + IRBuilder<> Builder(II); + return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_prefetch_tensormap, + NewV); + } } return nullptr; } diff --git a/llvm/lib/Target/PowerPC/PPCCCState.h b/llvm/lib/Target/PowerPC/PPCCCState.h index b0e50b2..feab9c5 100644 --- a/llvm/lib/Target/PowerPC/PPCCCState.h +++ b/llvm/lib/Target/PowerPC/PPCCCState.h @@ -38,36 +38,6 @@ public: void clearWasPPCF128() { OriginalArgWasPPCF128.clear(); } }; -class AIXCCState : public CCState { -private: - BitVector IsFixed; - -public: - AIXCCState(CallingConv::ID CC, bool IsVarArg, MachineFunction &MF, - SmallVectorImpl<CCValAssign> &Locs, LLVMContext &C) - : CCState(CC, IsVarArg, MF, Locs, C) {} - - void AnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins, - CCAssignFn Fn) { - // All formal arguments are fixed. - IsFixed.resize(Ins.size(), true); - CCState::AnalyzeFormalArguments(Ins, Fn); - } - - void AnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs, - CCAssignFn Fn) { - // Record whether the call operand was a fixed argument. - IsFixed.resize(Outs.size(), false); - for (unsigned ValNo = 0, E = Outs.size(); ValNo != E; ++ValNo) - if (Outs[ValNo].IsFixed) - IsFixed.set(ValNo); - - CCState::AnalyzeCallOperands(Outs, Fn); - } - - bool isFixed(unsigned ValNo) const { return IsFixed.test(ValNo); } -}; - } // end namespace llvm #endif diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index 196574e..2698bd6 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -6089,7 +6089,7 @@ SDValue PPCTargetLowering::LowerCall_32SVR4( ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; bool Result; - if (Outs[i].IsFixed) { + if (!ArgFlags.isVarArg()) { Result = CC_PPC32_SVR4(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo); } else { @@ -6905,8 +6905,7 @@ static bool isGPRShadowAligned(MCPhysReg Reg, Align RequiredAlign) { static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, - CCState &S) { - AIXCCState &State = static_cast<AIXCCState &>(S); + CCState &State) { const PPCSubtarget &Subtarget = static_cast<const PPCSubtarget &>( State.getMachineFunction().getSubtarget()); const bool IsPPC64 = Subtarget.isPPC64(); @@ -7090,7 +7089,7 @@ static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, // They are passed in VRs if any are available (unlike arguments passed // through ellipses) and shadow GPRs (unlike arguments to non-vaarg // functions) - if (State.isFixed(ValNo)) { + if (!ArgFlags.isVarArg()) { if (MCRegister VReg = State.AllocateReg(VR)) { State.addLoc(CCValAssign::getReg(ValNo, ValVT, VReg, LocVT, LocInfo)); // Shadow allocate GPRs and stack space even though we pass in a VR. @@ -7278,7 +7277,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); - AIXCCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); + CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); const EVT PtrVT = getPointerTy(MF.getDataLayout()); // Reserve space for the linkage area on the stack. @@ -7625,8 +7624,8 @@ SDValue PPCTargetLowering::LowerCall_AIX( MachineFunction &MF = DAG.getMachineFunction(); SmallVector<CCValAssign, 16> ArgLocs; - AIXCCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs, - *DAG.getContext()); + CCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs, + *DAG.getContext()); // Reserve space for the linkage save area (LSA) on the stack. // In both PPC32 and PPC64 there are 6 reserved slots in the LSA: diff --git a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp index d2b75a6..34026ed 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp +++ b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp @@ -45,8 +45,8 @@ public: CCValAssign::LocInfo LocInfo, const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags, CCState &State) override { - if (RISCVAssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State, Info.IsFixed, - IsRet, Info.Ty)) + if (RISCVAssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State, IsRet, + Info.Ty)) return true; StackSize = State.getStackSize(); @@ -196,8 +196,8 @@ public: if (LocVT.isScalableVector()) MF.getInfo<RISCVMachineFunctionInfo>()->setIsVectorCall(); - if (RISCVAssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State, - /*IsFixed=*/true, IsRet, Info.Ty)) + if (RISCVAssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State, IsRet, + Info.Ty)) return true; StackSize = State.getStackSize(); @@ -454,7 +454,7 @@ bool RISCVCallLowering::canLowerReturn(MachineFunction &MF, for (unsigned I = 0, E = Outs.size(); I < E; ++I) { MVT VT = MVT::getVT(Outs[I].Ty); if (CC_RISCV(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo, - /*IsFixed=*/true, /*isRet=*/true, nullptr)) + /*isRet=*/true, nullptr)) return false; } return true; diff --git a/llvm/lib/Target/RISCV/RISCVCallingConv.cpp b/llvm/lib/Target/RISCV/RISCVCallingConv.cpp index cb6117e..70127e3 100644 --- a/llvm/lib/Target/RISCV/RISCVCallingConv.cpp +++ b/llvm/lib/Target/RISCV/RISCVCallingConv.cpp @@ -324,7 +324,7 @@ static MCRegister allocateRVVReg(MVT ValVT, unsigned ValNo, CCState &State, // Implements the RISC-V calling convention. Returns true upon failure. bool llvm::CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, - CCState &State, bool IsFixed, bool IsRet, Type *OrigTy) { + CCState &State, bool IsRet, Type *OrigTy) { const MachineFunction &MF = State.getMachineFunction(); const DataLayout &DL = MF.getDataLayout(); const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>(); @@ -379,12 +379,12 @@ bool llvm::CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT, break; case RISCVABI::ABI_ILP32F: case RISCVABI::ABI_LP64F: - UseGPRForF16_F32 = !IsFixed; + UseGPRForF16_F32 = ArgFlags.isVarArg(); break; case RISCVABI::ABI_ILP32D: case RISCVABI::ABI_LP64D: - UseGPRForF16_F32 = !IsFixed; - UseGPRForF64 = !IsFixed; + UseGPRForF16_F32 = ArgFlags.isVarArg(); + UseGPRForF64 = ArgFlags.isVarArg(); break; } @@ -465,7 +465,7 @@ bool llvm::CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT, // currently if we are using ILP32E calling convention. This behavior may be // changed when RV32E/ILP32E is ratified. unsigned TwoXLenInBytes = (2 * XLen) / 8; - if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes && + if (ArgFlags.isVarArg() && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes && DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes && ABI != RISCVABI::ABI_ILP32E) { unsigned RegIdx = State.getFirstUnallocated(ArgGPRs); @@ -620,8 +620,8 @@ bool llvm::CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT, // benchmark. But theoretically, it may have benefit for some cases. bool llvm::CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, - ISD::ArgFlagsTy ArgFlags, CCState &State, - bool IsFixed, bool IsRet, Type *OrigTy) { + ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsRet, + Type *OrigTy) { const MachineFunction &MF = State.getMachineFunction(); const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>(); const RISCVTargetLowering &TLI = *Subtarget.getTargetLowering(); diff --git a/llvm/lib/Target/RISCV/RISCVCallingConv.h b/llvm/lib/Target/RISCV/RISCVCallingConv.h index bf823b7..2030ce1 100644 --- a/llvm/lib/Target/RISCV/RISCVCallingConv.h +++ b/llvm/lib/Target/RISCV/RISCVCallingConv.h @@ -21,15 +21,15 @@ namespace llvm { typedef bool RISCVCCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, - bool IsFixed, bool IsRet, Type *OrigTy); + bool IsRet, Type *OrigTy); bool CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, - CCState &State, bool IsFixed, bool IsRet, Type *OrigTy); + CCState &State, bool IsRet, Type *OrigTy); bool CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, - CCState &State, bool IsFixed, bool IsRet, Type *OrigTy); + CCState &State, bool IsRet, Type *OrigTy); bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 03e54b3..e4aa8b8 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -22282,8 +22282,8 @@ void RISCVTargetLowering::analyzeInputArgs( else if (In.isOrigArg()) ArgTy = FType->getParamType(In.getOrigArgIndex()); - if (Fn(Idx, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo, - /*IsFixed=*/true, IsRet, ArgTy)) { + if (Fn(Idx, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo, IsRet, + ArgTy)) { LLVM_DEBUG(dbgs() << "InputArg #" << Idx << " has unhandled type " << ArgVT << '\n'); llvm_unreachable(nullptr); @@ -22300,8 +22300,8 @@ void RISCVTargetLowering::analyzeOutputArgs( ISD::ArgFlagsTy ArgFlags = Out.Flags; Type *OrigTy = CLI ? CLI->getArgs()[Out.OrigArgIndex].Ty : nullptr; - if (Fn(Idx, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo, Out.IsFixed, - IsRet, OrigTy)) { + if (Fn(Idx, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo, IsRet, + OrigTy)) { LLVM_DEBUG(dbgs() << "OutputArg #" << Idx << " has unhandled type " << ArgVT << "\n"); llvm_unreachable(nullptr); @@ -23083,7 +23083,7 @@ bool RISCVTargetLowering::CanLowerReturn( MVT VT = Outs[i].VT; ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; if (CC_RISCV(i, VT, VT, CCValAssign::Full, ArgFlags, CCInfo, - /*IsFixed=*/true, /*IsRet=*/true, nullptr)) + /*IsRet=*/true, nullptr)) return false; } return true; diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp index 1aa8efe..c0fc3a6 100644 --- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp +++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp @@ -1179,7 +1179,7 @@ static void fixupVariableFloatArgs(SmallVectorImpl<CCValAssign> &ArgLocs, if (!VA.isRegLoc() || (ValTy != MVT::f64 && ValTy != MVT::f128)) continue; // The fixed arguments to a varargs function still go in FP registers. - if (Outs[VA.getValNo()].IsFixed) + if (!Outs[VA.getValNo()].Flags.isVarArg()) continue; // This floating point argument should be reassigned. diff --git a/llvm/lib/Target/SystemZ/SystemZCallingConv.h b/llvm/lib/Target/SystemZ/SystemZCallingConv.h index 25f4aac..fbb98ff 100644 --- a/llvm/lib/Target/SystemZ/SystemZCallingConv.h +++ b/llvm/lib/Target/SystemZ/SystemZCallingConv.h @@ -31,10 +31,6 @@ namespace SystemZ { class SystemZCCState : public CCState { private: - /// Records whether the value was a fixed argument. - /// See ISD::OutputArg::IsFixed. - SmallVector<bool, 4> ArgIsFixed; - /// Records whether the value was widened from a short vector type. SmallVector<bool, 4> ArgIsShortVector; @@ -50,10 +46,6 @@ public: void AnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins, CCAssignFn Fn) { - // Formal arguments are always fixed. - ArgIsFixed.clear(); - for (unsigned i = 0; i < Ins.size(); ++i) - ArgIsFixed.push_back(true); // Record whether the call operand was a short vector. ArgIsShortVector.clear(); for (unsigned i = 0; i < Ins.size(); ++i) @@ -64,10 +56,6 @@ public: void AnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs, CCAssignFn Fn) { - // Record whether the call operand was a fixed argument. - ArgIsFixed.clear(); - for (unsigned i = 0; i < Outs.size(); ++i) - ArgIsFixed.push_back(Outs[i].IsFixed); // Record whether the call operand was a short vector. ArgIsShortVector.clear(); for (unsigned i = 0; i < Outs.size(); ++i) @@ -77,12 +65,11 @@ public: } // This version of AnalyzeCallOperands in the base class is not usable - // since we must provide a means of accessing ISD::OutputArg::IsFixed. + // since we must provide a means of accessing ISD::OutputArg::IsShortVector. void AnalyzeCallOperands(const SmallVectorImpl<MVT> &Outs, SmallVectorImpl<ISD::ArgFlagsTy> &Flags, CCAssignFn Fn) = delete; - bool IsFixed(unsigned ValNo) { return ArgIsFixed[ValNo]; } bool IsShortVector(unsigned ValNo) { return ArgIsShortVector[ValNo]; } }; diff --git a/llvm/lib/Target/SystemZ/SystemZCallingConv.td b/llvm/lib/Target/SystemZ/SystemZCallingConv.td index 0ad872b..059f31f 100644 --- a/llvm/lib/Target/SystemZ/SystemZCallingConv.td +++ b/llvm/lib/Target/SystemZ/SystemZCallingConv.td @@ -16,14 +16,6 @@ class CCIfSubtarget<string F, CCAction A> "getSubtarget<SystemZSubtarget>().", F), A>; -// Match if this specific argument is a fixed (i.e. named) argument. -class CCIfFixed<CCAction A> - : CCIf<"static_cast<SystemZCCState *>(&State)->IsFixed(ValNo)", A>; - -// Match if this specific argument is not a fixed (i.e. vararg) argument. -class CCIfNotFixed<CCAction A> - : CCIf<"!(static_cast<SystemZCCState *>(&State)->IsFixed(ValNo))", A>; - // Match if this specific argument was widened from a short vector type. class CCIfShortVector<CCAction A> : CCIf<"static_cast<SystemZCCState *>(&State)->IsShortVector(ValNo)", A>; @@ -79,7 +71,7 @@ def CC_SystemZ_GHC : CallingConv<[ // Pass in STG registers: XMM1, ..., XMM6 CCIfSubtarget<"hasVector()", CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64], - CCIfFixed<CCAssignToReg<[V16, V17, V18, V19, V20, V21]>>>>, + CCIfArgFixed<CCAssignToReg<[V16, V17, V18, V19, V20, V21]>>>>, // Fail otherwise CCCustom<"CC_SystemZ_GHC_Error"> @@ -125,8 +117,8 @@ def CC_SystemZ_ELF : CallingConv<[ // during type legalization. CCIfSubtarget<"hasVector()", CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64], - CCIfFixed<CCAssignToReg<[V24, V26, V28, V30, - V25, V27, V29, V31]>>>>, + CCIfArgFixed<CCAssignToReg<[V24, V26, V28, V30, + V25, V27, V29, V31]>>>>, // However, sub-128 vectors which need to go on the stack occupy just a // single 8-byte-aligned 8-byte stack slot. Pass as i64. @@ -227,17 +219,17 @@ def CC_SystemZ_XPLINK64 : CallingConv<[ // Promote f32 to f64 and bitcast to i64, if it needs to be passed in GPRs. // Although we assign the f32 vararg to be bitcast, it will first be promoted // to an f64 within convertValVTToLocVT(). - CCIfType<[f32, f64], CCIfNotFixed<CCBitConvertToType<i64>>>, + CCIfType<[f32, f64], CCIfArgVarArg<CCBitConvertToType<i64>>>, // Pointers are always passed in full 64-bit registers. CCIfPtr<CCCustom<"CC_XPLINK64_Pointer">>, // long double, can only be passed in GPR2 and GPR3, if available, // hence R2Q - CCIfType<[f128], CCIfNotFixed<CCCustom<"CC_XPLINK64_Allocate128BitVararg">>>, + CCIfType<[f128], CCIfArgVarArg<CCCustom<"CC_XPLINK64_Allocate128BitVararg">>>, // Non fixed vector arguments are treated in the same way as long // doubles. CCIfSubtarget<"hasVector()", CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64], - CCIfNotFixed<CCCustom<"CC_XPLINK64_Allocate128BitVararg">>>>, + CCIfArgVarArg<CCCustom<"CC_XPLINK64_Allocate128BitVararg">>>>, // A SwiftSelf is passed in callee-saved R10. CCIfSwiftSelf<CCIfType<[i64], CCAssignToReg<[R10D]>>>, @@ -260,22 +252,24 @@ def CC_SystemZ_XPLINK64 : CallingConv<[ // during type legalization. CCIfSubtarget<"hasVector()", CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64], - CCIfFixed<CCCustom<"CC_XPLINK64_Shadow_Reg">>>>, + CCIfArgFixed<CCCustom<"CC_XPLINK64_Shadow_Reg">>>>, CCIfSubtarget<"hasVector()", CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64], - CCIfFixed<CCAssignToRegAndStack<[V24, V25, V26, V27, - V28, V29, V30, V31], 16, 8>>>>, + CCIfArgFixed<CCAssignToRegAndStack<[V24, V25, V26, V27, + V28, V29, V30, V31], 16, 8>>>>, // The first 4 named float and double arguments are passed in registers // FPR0-FPR6. The rest will be passed in the user area. - CCIfType<[f32, f64], CCIfFixed<CCCustom<"CC_XPLINK64_Shadow_Reg">>>, - CCIfType<[f32], CCIfFixed<CCAssignToRegAndStack<[F0S, F2S, F4S, F6S], 4, 8>>>, - CCIfType<[f64], CCIfFixed<CCAssignToRegAndStack<[F0D, F2D, F4D, F6D], 8, 8>>>, + CCIfType<[f32, f64], CCIfArgFixed<CCCustom<"CC_XPLINK64_Shadow_Reg">>>, + CCIfType<[f32], + CCIfArgFixed<CCAssignToRegAndStack<[F0S, F2S, F4S, F6S], 4, 8>>>, + CCIfType<[f64], + CCIfArgFixed<CCAssignToRegAndStack<[F0D, F2D, F4D, F6D], 8, 8>>>, // The first 2 long double arguments are passed in register FPR0/FPR2 // and FPR4/FPR6. The rest will be passed in the user area. - CCIfType<[f128], CCIfFixed<CCCustom<"CC_XPLINK64_Shadow_Reg">>>, - CCIfType<[f128], CCIfFixed<CCAssignToRegAndStack<[F0Q, F4Q], 16, 8>>>, + CCIfType<[f128], CCIfArgFixed<CCCustom<"CC_XPLINK64_Shadow_Reg">>>, + CCIfType<[f128], CCIfArgFixed<CCAssignToRegAndStack<[F0Q, F4Q], 16, 8>>>, // Other arguments are passed in 8-byte-aligned 8-byte stack slots. CCIfType<[i32, i64, f32, f64], CCAssignToStack<8, 8>>, diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp index 3f80b2a..f9eba4b 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp @@ -1309,7 +1309,7 @@ WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI, OutVal = FINode; } // Count the number of fixed args *after* legalization. - NumFixedArgs += Out.IsFixed; + NumFixedArgs += !Out.Flags.isVarArg(); } bool IsVarArg = CLI.IsVarArg; @@ -1503,7 +1503,7 @@ SDValue WebAssemblyTargetLowering::LowerReturn( for (const ISD::OutputArg &Out : Outs) { assert(!Out.Flags.isByVal() && "byval is not valid for return values"); assert(!Out.Flags.isNest() && "nest is not valid for return values"); - assert(Out.IsFixed && "non-fixed return value is not valid"); + assert(!Out.Flags.isVarArg() && "non-fixed return value is not valid"); if (Out.Flags.isInAlloca()) fail(DL, DAG, "WebAssembly hasn't implemented inalloca results"); if (Out.Flags.isInConsecutiveRegs()) diff --git a/llvm/lib/Target/X86/GISel/X86CallLowering.cpp b/llvm/lib/Target/X86/GISel/X86CallLowering.cpp index c0a6035..d9f4405 100644 --- a/llvm/lib/Target/X86/GISel/X86CallLowering.cpp +++ b/llvm/lib/Target/X86/GISel/X86CallLowering.cpp @@ -75,7 +75,7 @@ public: static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7}; - if (!Info.IsFixed) + if (Flags.isVarArg()) NumXMMRegs = State.getFirstUnallocated(XMMArgRegs); return Res; @@ -363,7 +363,8 @@ bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, Info.CallConv, Info.IsVarArg)) return false; - bool IsFixed = Info.OrigArgs.empty() ? true : Info.OrigArgs.back().IsFixed; + bool IsFixed = + Info.OrigArgs.empty() ? true : !Info.OrigArgs.back().Flags[0].isVarArg(); if (STI.is64Bit() && !IsFixed && !STI.isCallingConvWin64(Info.CallConv)) { // From AMD64 ABI document: // For calls that may call functions that use varargs or stdargs diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index ed3dca2..59a47a9 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -2361,15 +2361,13 @@ remapIndices(Function &Caller, BasicBlock *StartBB, // Updating the contextual profile after an inlining means, at a high level, // copying over the data of the callee, **intentionally without any value // scaling**, and copying over the callees of the inlined callee. -llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, - PGOContextualProfile &CtxProf, - bool MergeAttributes, - AAResults *CalleeAAR, - bool InsertLifetime, - Function *ForwardVarArgsTo) { +llvm::InlineResult llvm::InlineFunction( + CallBase &CB, InlineFunctionInfo &IFI, PGOContextualProfile &CtxProf, + bool MergeAttributes, AAResults *CalleeAAR, bool InsertLifetime, + Function *ForwardVarArgsTo, OptimizationRemarkEmitter *ORE) { if (!CtxProf.isInSpecializedModule()) return InlineFunction(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime, - ForwardVarArgsTo); + ForwardVarArgsTo, ORE); auto &Caller = *CB.getCaller(); auto &Callee = *CB.getCalledFunction(); @@ -2387,7 +2385,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, const auto NumCalleeCallsites = CtxProf.getNumCallsites(Callee); auto Ret = InlineFunction(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime, - ForwardVarArgsTo); + ForwardVarArgsTo, ORE); if (!Ret.isSuccess()) return Ret; @@ -2457,20 +2455,8 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, return Ret; } -/// This function inlines the called function into the basic block of the -/// caller. This returns false if it is not possible to inline this call. -/// The program is still in a well defined state if this occurs though. -/// -/// Note that this only does one level of inlining. For example, if the -/// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now -/// exists in the instruction stream. Similarly this will inline a recursive -/// function by one level. -llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, - bool MergeAttributes, - AAResults *CalleeAAR, - bool InsertLifetime, - Function *ForwardVarArgsTo, - OptimizationRemarkEmitter *ORE) { +llvm::InlineResult llvm::CanInlineCallSite(const CallBase &CB, + InlineFunctionInfo &IFI) { assert(CB.getParent() && CB.getFunction() && "Instruction not in function!"); // FIXME: we don't inline callbr yet. @@ -2487,7 +2473,6 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, // The inliner does not know how to inline through calls with operand bundles // in general ... - Value *ConvergenceControlToken = nullptr; if (CB.hasOperandBundles()) { for (int i = 0, e = CB.getNumOperandBundles(); i != e; ++i) { auto OBUse = CB.getOperandBundleAt(i); @@ -2503,7 +2488,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, if (Tag == LLVMContext::OB_kcfi) continue; if (Tag == LLVMContext::OB_convergencectrl) { - ConvergenceControlToken = OBUse.Inputs[0].get(); + IFI.ConvergenceControlToken = OBUse.Inputs[0].get(); continue; } @@ -2521,28 +2506,22 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, // fully implements convergence control tokens, there is no mixing of // controlled and uncontrolled convergent operations in the whole program. if (CB.isConvergent()) { - if (!ConvergenceControlToken && + if (!IFI.ConvergenceControlToken && getConvergenceEntry(CalledFunc->getEntryBlock())) { return InlineResult::failure( "convergent call needs convergencectrl operand"); } } - // If the call to the callee cannot throw, set the 'nounwind' flag on any - // calls that we inline. - bool MarkNoUnwind = CB.doesNotThrow(); - - BasicBlock *OrigBB = CB.getParent(); - Function *Caller = OrigBB->getParent(); + const BasicBlock *OrigBB = CB.getParent(); + const Function *Caller = OrigBB->getParent(); // GC poses two hazards to inlining, which only occur when the callee has GC: // 1. If the caller has no GC, then the callee's GC must be propagated to the // caller. // 2. If the caller has a differing GC, it is invalid to inline. if (CalledFunc->hasGC()) { - if (!Caller->hasGC()) - Caller->setGC(CalledFunc->getGC()); - else if (CalledFunc->getGC() != Caller->getGC()) + if (Caller->hasGC() && CalledFunc->getGC() != Caller->getGC()) return InlineResult::failure("incompatible GC"); } @@ -2560,34 +2539,31 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, ? Caller->getPersonalityFn()->stripPointerCasts() : nullptr; if (CalledPersonality) { - if (!CallerPersonality) - Caller->setPersonalityFn(CalledPersonality); // If the personality functions match, then we can perform the // inlining. Otherwise, we can't inline. // TODO: This isn't 100% true. Some personality functions are proper // supersets of others and can be used in place of the other. - else if (CalledPersonality != CallerPersonality) + if (CallerPersonality && CalledPersonality != CallerPersonality) return InlineResult::failure("incompatible personality"); } // We need to figure out which funclet the callsite was in so that we may // properly nest the callee. - Instruction *CallSiteEHPad = nullptr; if (CallerPersonality) { EHPersonality Personality = classifyEHPersonality(CallerPersonality); if (isScopedEHPersonality(Personality)) { std::optional<OperandBundleUse> ParentFunclet = CB.getOperandBundle(LLVMContext::OB_funclet); if (ParentFunclet) - CallSiteEHPad = cast<FuncletPadInst>(ParentFunclet->Inputs.front()); + IFI.CallSiteEHPad = cast<FuncletPadInst>(ParentFunclet->Inputs.front()); // OK, the inlining site is legal. What about the target function? - if (CallSiteEHPad) { + if (IFI.CallSiteEHPad) { if (Personality == EHPersonality::MSVC_CXX) { // The MSVC personality cannot tolerate catches getting inlined into // cleanup funclets. - if (isa<CleanupPadInst>(CallSiteEHPad)) { + if (isa<CleanupPadInst>(IFI.CallSiteEHPad)) { // Ok, the call site is within a cleanuppad. Let's check the callee // for catchpads. for (const BasicBlock &CalledBB : *CalledFunc) { @@ -2607,13 +2583,34 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, } } + return InlineResult::success(); +} + +/// This function inlines the called function into the basic block of the +/// caller. This returns false if it is not possible to inline this call. +/// The program is still in a well defined state if this occurs though. +/// +/// Note that this only does one level of inlining. For example, if the +/// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now +/// exists in the instruction stream. Similarly this will inline a recursive +/// function by one level. +void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI, + bool MergeAttributes, AAResults *CalleeAAR, + bool InsertLifetime, Function *ForwardVarArgsTo, + OptimizationRemarkEmitter *ORE) { + BasicBlock *OrigBB = CB.getParent(); + Function *Caller = OrigBB->getParent(); + Function *CalledFunc = CB.getCalledFunction(); + assert(CalledFunc && !CalledFunc->isDeclaration() && + "CanInlineCallSite should have verified direct call to definition"); + // Determine if we are dealing with a call in an EHPad which does not unwind // to caller. bool EHPadForCallUnwindsLocally = false; - if (CallSiteEHPad && isa<CallInst>(CB)) { + if (IFI.CallSiteEHPad && isa<CallInst>(CB)) { UnwindDestMemoTy FuncletUnwindMap; Value *CallSiteUnwindDestToken = - getUnwindDestToken(CallSiteEHPad, FuncletUnwindMap); + getUnwindDestToken(IFI.CallSiteEHPad, FuncletUnwindMap); EHPadForCallUnwindsLocally = CallSiteUnwindDestToken && @@ -2630,6 +2627,30 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, ClonedCodeInfo InlinedFunctionInfo; Function::iterator FirstNewBlock; + // GC poses two hazards to inlining, which only occur when the callee has GC: + // 1. If the caller has no GC, then the callee's GC must be propagated to the + // caller. + // 2. If the caller has a differing GC, it is invalid to inline. + if (CalledFunc->hasGC()) { + if (!Caller->hasGC()) + Caller->setGC(CalledFunc->getGC()); + else { + assert(CalledFunc->getGC() == Caller->getGC() && + "CanInlineCallSite should have verified compatible GCs"); + } + } + + if (CalledFunc->hasPersonalityFn()) { + Constant *CalledPersonality = + CalledFunc->getPersonalityFn()->stripPointerCasts(); + if (!Caller->hasPersonalityFn()) { + Caller->setPersonalityFn(CalledPersonality); + } else + assert(Caller->getPersonalityFn()->stripPointerCasts() == + CalledPersonality && + "CanInlineCallSite should have verified compatible personality"); + } + { // Scope to destroy VMap after cloning. ValueToValueMapTy VMap; struct ByValInit { @@ -2819,10 +2840,10 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, IFI.GetAssumptionCache(*Caller).registerAssumption(II); } - if (ConvergenceControlToken) { + if (IFI.ConvergenceControlToken) { IntrinsicInst *IntrinsicCall = getConvergenceEntry(*FirstNewBlock); if (IntrinsicCall) { - IntrinsicCall->replaceAllUsesWith(ConvergenceControlToken); + IntrinsicCall->replaceAllUsesWith(IFI.ConvergenceControlToken); IntrinsicCall->eraseFromParent(); } } @@ -2869,6 +2890,10 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, } } + // If the call to the callee cannot throw, set the 'nounwind' flag on any + // calls that we inline. + bool MarkNoUnwind = CB.doesNotThrow(); + SmallVector<Value*,4> VarArgsToForward; SmallVector<AttributeSet, 4> VarArgsAttrs; for (unsigned i = CalledFunc->getFunctionType()->getNumParams(); @@ -3055,12 +3080,12 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, // Update the lexical scopes of the new funclets and callsites. // Anything that had 'none' as its parent is now nested inside the callsite's // EHPad. - if (CallSiteEHPad) { + if (IFI.CallSiteEHPad) { for (Function::iterator BB = FirstNewBlock->getIterator(), E = Caller->end(); BB != E; ++BB) { // Add bundle operands to inlined call sites. - PropagateOperandBundles(BB, CallSiteEHPad); + PropagateOperandBundles(BB, IFI.CallSiteEHPad); // It is problematic if the inlinee has a cleanupret which unwinds to // caller and we inline it into a call site which doesn't unwind but into @@ -3076,11 +3101,11 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(I)) { if (isa<ConstantTokenNone>(CatchSwitch->getParentPad())) - CatchSwitch->setParentPad(CallSiteEHPad); + CatchSwitch->setParentPad(IFI.CallSiteEHPad); } else { auto *FPI = cast<FuncletPadInst>(I); if (isa<ConstantTokenNone>(FPI->getParentPad())) - FPI->setParentPad(CallSiteEHPad); + FPI->setParentPad(IFI.CallSiteEHPad); } } } @@ -3236,7 +3261,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, AttributeFuncs::mergeAttributesForInlining(*Caller, *CalledFunc); // We are now done with the inlining. - return InlineResult::success(); + return; } // Otherwise, we have the normal case, of more than one block to inline or @@ -3404,6 +3429,19 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, if (MergeAttributes) AttributeFuncs::mergeAttributesForInlining(*Caller, *CalledFunc); +} - return InlineResult::success(); +llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, + bool MergeAttributes, + AAResults *CalleeAAR, + bool InsertLifetime, + Function *ForwardVarArgsTo, + OptimizationRemarkEmitter *ORE) { + llvm::InlineResult Result = CanInlineCallSite(CB, IFI); + if (Result.isSuccess()) { + InlineFunctionImpl(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime, + ForwardVarArgsTo, ORE); + } + + return Result; } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 323e422..be00fd6 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7530,9 +7530,6 @@ BasicBlock *EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton() { EPI.MainLoopIterationCountCheck = emitIterationCountCheck(LoopScalarPreHeader, false); - // Generate the induction variable. - EPI.VectorTripCount = getOrCreateVectorTripCount(LoopVectorPreHeader); - replaceVPBBWithIRVPBB(Plan.getScalarPreheader(), LoopScalarPreHeader); return LoopVectorPreHeader; } @@ -9808,7 +9805,7 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) { static void preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs, - const EpilogueLoopVectorizationInfo &EPI) { + EpilogueLoopVectorizationInfo &EPI) { VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion(); VPBasicBlock *Header = VectorLoop->getEntryBasicBlock(); Header->setName("vec.epilog.vector.body"); @@ -9826,30 +9823,13 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, // loop. using namespace llvm::PatternMatch; PHINode *EPResumeVal = &*L->getLoopPreheader()->phis().begin(); - assert(EPResumeVal->getType() == IV->getScalarType() && - match(EPResumeVal->getIncomingValueForBlock( - EPI.MainLoopIterationCountCheck), - m_SpecificInt(0)) && - EPResumeVal == - find_singleton<PHINode>( - L->getLoopPreheader()->phis(), - [&EPI, IV](PHINode &P, bool) -> PHINode * { - if (P.getType() == IV->getScalarType() && - match(P.getIncomingValueForBlock( - EPI.MainLoopIterationCountCheck), - m_SpecificInt(0)) && - any_of(P.incoming_values(), - [&EPI](Value *Inc) { - return Inc == EPI.VectorTripCount; - }) && - all_of(P.incoming_values(), [&EPI](Value *Inc) { - return Inc == EPI.VectorTripCount || - match(Inc, m_SpecificInt(0)); - })) - return &P; - return nullptr; - }) && - "Epilogue resume phis do not match!"); + for (Value *Inc : EPResumeVal->incoming_values()) { + if (match(Inc, m_SpecificInt(0))) + continue; + assert(!EPI.VectorTripCount && + "Must only have a single non-zero incoming value"); + EPI.VectorTripCount = Inc; + } VPValue *VPV = Plan.getOrAddLiveIn(EPResumeVal); assert(all_of(IV->users(), [](const VPUser *U) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 8052e31..73babcc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1054,12 +1054,17 @@ void VPlan::execute(VPTransformState *State) { InstructionCost VPlan::cost(ElementCount VF, VPCostContext &Ctx) { // For now only return the cost of the vector loop region, ignoring any other - // blocks, like the preheader or middle blocks. + // blocks, like the preheader or middle blocks, expect for checking them for + // recipes with invalid costs. InstructionCost Cost = getVectorLoopRegion()->cost(VF, Ctx); - // If any instructions in the middle block are invalid return invalid. - // TODO: Remove once no VPlans with VF == vscale x 1 and first-order recurrences are created. - if (!getMiddleBlock()->cost(VF, Ctx).isValid()) + // If the cost of the loop region is invalid or any recipe in the skeleton + // outside loop regions are invalid return an invalid cost. + if (!Cost.isValid() || any_of(VPBlockUtils::blocksOnly<VPBasicBlock>( + vp_depth_first_shallow(getEntry())), + [&VF, &Ctx](VPBasicBlock *VPBB) { + return !VPBB->cost(VF, Ctx).isValid(); + })) return InstructionCost::getInvalid(); return Cost; diff --git a/llvm/test/CodeGen/AArch64/aarch64-split-and-bitmask-immediate.ll b/llvm/test/CodeGen/AArch64/aarch64-split-logic-bitmask-immediate.ll index 113eb14..4db9db9 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-split-and-bitmask-immediate.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-split-logic-bitmask-immediate.ll @@ -370,3 +370,175 @@ entry: %r = select i1 %c, i64 %a, i64 %ands ret i64 %r } + +; Test EOR. +define i32 @test1_eor(i32 %a) { +; CHECK-LABEL: test1_eor: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: eor w8, w0, #0x400 +; CHECK-NEXT: eor w0, w8, #0x200000 +; CHECK-NEXT: ret +entry: + %eor = xor i32 %a, 2098176 + ret i32 %eor +} + +; This constant should not be split because it can be handled by one mov. +define i32 @test2_eor(i32 %a) { +; CHECK-LABEL: test2_eor: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov w8, #135 // =0x87 +; CHECK-NEXT: eor w0, w0, w8 +; CHECK-NEXT: ret +entry: + %eor = xor i32 %a, 135 + ret i32 %eor +} + +; This constant should not be split because the split immediate is not valid +; bitmask immediate. +define i32 @test3_eor(i32 %a) { +; CHECK-LABEL: test3_eor: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov w8, #1024 // =0x400 +; CHECK-NEXT: movk w8, #33, lsl #16 +; CHECK-NEXT: eor w0, w0, w8 +; CHECK-NEXT: ret +entry: + %eor = xor i32 %a, 2163712 + ret i32 %eor +} + +define i64 @test4_eor(i64 %a) { +; CHECK-LABEL: test4_eor: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: eor x8, x0, #0x400 +; CHECK-NEXT: eor x0, x8, #0x200000 +; CHECK-NEXT: ret +entry: + %eor = xor i64 %a, 2098176 + ret i64 %eor +} + +define i64 @test5_eor(i64 %a) { +; CHECK-LABEL: test5_eor: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: eor x8, x0, #0x4000 +; CHECK-NEXT: eor x0, x8, #0x200000000 +; CHECK-NEXT: ret +entry: + %eor = xor i64 %a, 8589950976 + ret i64 %eor +} + +; This constant should not be split because it can be handled by one mov. +define i64 @test6_eor(i64 %a) { +; CHECK-LABEL: test6_eor: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov w8, #135 // =0x87 +; CHECK-NEXT: eor x0, x0, x8 +; CHECK-NEXT: ret +entry: + %eor = xor i64 %a, 135 + ret i64 %eor +} + +; This constant should not be split because the split immediate is not valid +; bitmask immediate. +define i64 @test7_eor(i64 %a) { +; CHECK-LABEL: test7_eor: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov w8, #1024 // =0x400 +; CHECK-NEXT: movk w8, #33, lsl #16 +; CHECK-NEXT: eor x0, x0, x8 +; CHECK-NEXT: ret +entry: + %eor = xor i64 %a, 2163712 + ret i64 %eor +} + +; Test ORR. +define i32 @test1_orr(i32 %a) { +; CHECK-LABEL: test1_orr: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: orr w8, w0, #0x400 +; CHECK-NEXT: orr w0, w8, #0x200000 +; CHECK-NEXT: ret +entry: + %orr = or i32 %a, 2098176 + ret i32 %orr +} + +; This constant should not be split because it can be handled by one mov. +define i32 @test2_orr(i32 %a) { +; CHECK-LABEL: test2_orr: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov w8, #135 // =0x87 +; CHECK-NEXT: orr w0, w0, w8 +; CHECK-NEXT: ret +entry: + %orr = or i32 %a, 135 + ret i32 %orr +} + +; This constant should not be split because the split immediate is not valid +; bitmask immediate. +define i32 @test3_orr(i32 %a) { +; CHECK-LABEL: test3_orr: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov w8, #1024 // =0x400 +; CHECK-NEXT: movk w8, #33, lsl #16 +; CHECK-NEXT: orr w0, w0, w8 +; CHECK-NEXT: ret +entry: + %orr = or i32 %a, 2163712 + ret i32 %orr +} + +define i64 @test4_orr(i64 %a) { +; CHECK-LABEL: test4_orr: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: orr x8, x0, #0x400 +; CHECK-NEXT: orr x0, x8, #0x200000 +; CHECK-NEXT: ret +entry: + %orr = or i64 %a, 2098176 + ret i64 %orr +} + +define i64 @test5_orr(i64 %a) { +; CHECK-LABEL: test5_orr: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: orr x8, x0, #0x4000 +; CHECK-NEXT: orr x0, x8, #0x200000000 +; CHECK-NEXT: ret +entry: + %orr = or i64 %a, 8589950976 + ret i64 %orr +} + +; This constant should not be split because it can be handled by one mov. +define i64 @test6_orr(i64 %a) { +; CHECK-LABEL: test6_orr: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov w8, #135 // =0x87 +; CHECK-NEXT: orr x0, x0, x8 +; CHECK-NEXT: ret +entry: + %orr = or i64 %a, 135 + ret i64 %orr +} + +; This constant should not be split because the split immediate is not valid +; bitmask immediate. +define i64 @test7_orr(i64 %a) { +; CHECK-LABEL: test7_orr: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov w8, #1024 // =0x400 +; CHECK-NEXT: movk w8, #33, lsl #16 +; CHECK-NEXT: orr x0, x0, x8 +; CHECK-NEXT: ret +entry: + %orr = or i64 %a, 2163712 + ret i64 %orr +} diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir index c7767cb8..b53bde6 100644 --- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir +++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir @@ -20,11 +20,32 @@ ret void } + define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_physreg_src2() #0 { + ret void + } + define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_src2_different_subreg() #0 { ret void } + define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first() #1 { + ret void + } + + define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second() #1 { + ret void + } + + define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first_physreg() #1 { + ret void + } + + define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second_physreg() #1 { + ret void + } + attributes #0 = { "amdgpu-wave-limiter"="true" "amdgpu-waves-per-eu"="8,8" } + attributes #1 = { "amdgpu-wave-limiter"="true" "amdgpu-waves-per-eu"="10,10" } ... # Inflate pattern, except the defining instruction isn't an MFMA. @@ -403,6 +424,89 @@ body: | ... +# Non-mac variant, src2 is a physical register +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_physreg_src2 +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_physreg_src2 + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $vcc, $vgpr0_vgpr1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: early-clobber renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, undef $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17:0x00000000FFFFFFFF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + bb.0: + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + + bb.1: + liveins: $vcc + + %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, undef $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_VCCNZ %bb.1, implicit $vcc + S_BRANCH %bb.2 + + bb.2: + ; No VGPRs available for %0 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... + # Non-mac variant, src2 is the same VGPR, but a different subregister. --- name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_src2_different_subreg @@ -489,3 +593,423 @@ body: | S_ENDPGM 0 ... + +# There isn't an assignable AGPR around the first MFMA. +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $vcc, $vgpr18_vgpr19 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit-def renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit-def renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit-def renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit killed renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit killed renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit killed renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + ; CHECK-NEXT: early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + bb.0: + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + + bb.1: + liveins: $vcc + + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + S_NOP 0, implicit-def %6:areg_512_align2, implicit-def %7:areg_512_align2, implicit-def %8:areg_512_align2, implicit-def %9:areg_512_align2 + %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + S_NOP 0, implicit %6, implicit %7, implicit %8, implicit %9 + %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_VCCNZ %bb.1, implicit $vcc + S_BRANCH %bb.2 + + bb.2: + ; No VGPRs available for %0 or %4 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... + +# There isn't an assignable AGPR around the second MFMA. +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $vcc, $vgpr18_vgpr19 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit-def renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit-def renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit-def renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + ; CHECK-NEXT: early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit killed renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit killed renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit killed renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + bb.0: + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + + bb.1: + liveins: $vcc + + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + S_NOP 0, implicit-def %6:areg_512_align2, implicit-def %7:areg_512_align2, implicit-def %8:areg_512_align2, implicit-def %9:areg_512_align2 + %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec + S_NOP 0, implicit %6, implicit %7, implicit %8, implicit %9 + S_CBRANCH_VCCNZ %bb.1, implicit $vcc + S_BRANCH %bb.2 + + bb.2: + ; No VGPRs available for %0 or %4 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... + +# There isn't an assignable AGPR around the first MFMA, with physreg interference +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first_physreg +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first_physreg + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $vcc, $vgpr18_vgpr19 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_NOP 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 + ; CHECK-NEXT: S_NOP 0, implicit $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: S_NOP 0, implicit $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23 + ; CHECK-NEXT: S_NOP 0, implicit $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 + ; CHECK-NEXT: S_NOP 0, implicit $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39 + ; CHECK-NEXT: S_NOP 0, implicit $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47 + ; CHECK-NEXT: S_NOP 0, implicit $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55 + ; CHECK-NEXT: S_NOP 0, implicit $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + ; CHECK-NEXT: early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + bb.0: + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + + bb.1: + liveins: $vcc + + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + S_NOP 0, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 + S_NOP 0, implicit-def $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + S_NOP 0, implicit-def $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23 + S_NOP 0, implicit-def $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 + S_NOP 0, implicit-def $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39 + S_NOP 0, implicit-def $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47 + S_NOP 0, implicit-def $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55 + S_NOP 0, implicit-def $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + S_NOP 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 + S_NOP 0, implicit $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + S_NOP 0, implicit $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23 + S_NOP 0, implicit $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 + S_NOP 0, implicit $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39 + S_NOP 0, implicit $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47 + S_NOP 0, implicit $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55 + S_NOP 0, implicit $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_VCCNZ %bb.1, implicit $vcc + S_BRANCH %bb.2 + + bb.2: + ; No VGPRs available for %0 or %4 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... + +# There isn't an assignable AGPR around the second MFMA, physreg interference +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second_physreg +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second_physreg + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $vcc, $vgpr18_vgpr19 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + ; CHECK-NEXT: early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_NOP 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 + ; CHECK-NEXT: S_NOP 0, implicit $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: S_NOP 0, implicit $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23 + ; CHECK-NEXT: S_NOP 0, implicit $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 + ; CHECK-NEXT: S_NOP 0, implicit $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39 + ; CHECK-NEXT: S_NOP 0, implicit $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47 + ; CHECK-NEXT: S_NOP 0, implicit $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55 + ; CHECK-NEXT: S_NOP 0, implicit $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + bb.0: + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + + bb.1: + liveins: $vcc + + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + S_NOP 0, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 + S_NOP 0, implicit-def $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + S_NOP 0, implicit-def $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23 + S_NOP 0, implicit-def $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 + S_NOP 0, implicit-def $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39 + S_NOP 0, implicit-def $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47 + S_NOP 0, implicit-def $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55 + S_NOP 0, implicit-def $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec + S_NOP 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 + S_NOP 0, implicit $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + S_NOP 0, implicit $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23 + S_NOP 0, implicit $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 + S_NOP 0, implicit $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39 + S_NOP 0, implicit $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47 + S_NOP 0, implicit $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55 + S_NOP 0, implicit $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + S_CBRANCH_VCCNZ %bb.1, implicit $vcc + S_BRANCH %bb.2 + + bb.2: + ; No VGPRs available for %0 or %4 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir index b907c13..b59f2de 100644 --- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir +++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir @@ -445,6 +445,86 @@ body: | ... + +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses_cannot_rewrite_final_use +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses_cannot_rewrite_final_use + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $vcc, $vgpr0_vgpr1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, killed $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17:0x00000000FFFFFFFF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: INLINEASM &"; use $0 ", 1 /* sideeffect attdialect */, 27983881 /* reguse:VReg_512_Align2 */, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_ENDPGM 0 + bb.0: + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + + bb.1: + liveins: $vcc + + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_VCCNZ %bb.1, implicit $vcc + S_BRANCH %bb.2 + + bb.2: + ; No VGPRs available for %0 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + INLINEASM &"; use $0 ", 1 /* sideeffect attdialect */, 27983881 /* reguse:VReg_512_Align2 */, %0:vreg_512_align2 + S_ENDPGM 0 + +... + # There is a rewrite candidate, but it is used by another MFMA which # does not have a tied result. --- @@ -619,10 +699,9 @@ body: | S_ENDPGM 0 ... - -# There isn't an assignable AGPR around the first MFMA. +# Chain of 2 untied cases, but the use isn't in src2. --- -name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_non_src2 tracksRegLiveness: true machineFunctionInfo: isEntryFunction: true @@ -630,7 +709,7 @@ machineFunctionInfo: occupancy: 10 sgprForEXECCopy: '$sgpr100_sgpr101' body: | - ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_non_src2 ; CHECK: bb.0: ; CHECK-NEXT: successors: %bb.1(0x80000000) ; CHECK-NEXT: {{ $}} @@ -647,10 +726,8 @@ body: | ; CHECK-NEXT: liveins: $vcc, $vgpr18_vgpr19 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) - ; CHECK-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit-def renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit-def renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit-def renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec - ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit killed renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit killed renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit killed renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 - ; CHECK-NEXT: early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr4_vgpr5, $vgpr8_vgpr9, undef $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc ; CHECK-NEXT: S_BRANCH %bb.2 ; CHECK-NEXT: {{ $}} @@ -685,10 +762,8 @@ body: | liveins: $vcc undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) - S_NOP 0, implicit-def %6:areg_512_align2, implicit-def %7:areg_512_align2, implicit-def %8:areg_512_align2, implicit-def %9:areg_512_align2 %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec - S_NOP 0, implicit %6, implicit %7, implicit %8, implicit %9 - %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec + %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %3.sub4_sub5, %3.sub8_sub9, undef %6:vreg_512_align2, 0, 0, 0, implicit $mode, implicit $exec S_CBRANCH_VCCNZ %bb.1, implicit $vcc S_BRANCH %bb.2 @@ -711,9 +786,10 @@ body: | ... -# There isn't an assignable AGPR around the second MFMA. +# Chain of 2 untied cases, but the second mfma is a different size and +# uses a subregister. --- -name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_subreg tracksRegLiveness: true machineFunctionInfo: isEntryFunction: true @@ -721,7 +797,7 @@ machineFunctionInfo: occupancy: 10 sgprForEXECCopy: '$sgpr100_sgpr101' body: | - ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_subreg ; CHECK: bb.0: ; CHECK-NEXT: successors: %bb.1(0x80000000) ; CHECK-NEXT: {{ $}} @@ -739,18 +815,16 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec - ; CHECK-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit-def renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit-def renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit-def renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 - ; CHECK-NEXT: early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec - ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit killed renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit killed renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit killed renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 + ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr2_vgpr3_vgpr4_vgpr5, 0, 0, 0, implicit $mode, implicit $exec ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc ; CHECK-NEXT: S_BRANCH %bb.2 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: - ; CHECK-NEXT: liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF + ; CHECK-NEXT: liveins: $vgpr0_vgpr1_vgpr2_vgpr3 ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 - ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 @@ -758,10 +832,7 @@ body: | ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec - ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) - ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) - ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) - ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) ; CHECK-NEXT: S_ENDPGM 0 bb.0: S_NOP 0, implicit-def $agpr0 @@ -777,9 +848,7 @@ body: | undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec - S_NOP 0, implicit-def %6:areg_512_align2, implicit-def %7:areg_512_align2, implicit-def %8:areg_512_align2, implicit-def %9:areg_512_align2 - %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec - S_NOP 0, implicit %6, implicit %7, implicit %8, implicit %9 + %4:vreg_128_align2 = V_MFMA_F32_16X16X16F16_vgprcd_e64 %1, %1, %3.sub2_sub3_sub4_sub5, 0, 0, 0, implicit $mode, implicit $exec S_CBRANCH_VCCNZ %bb.1, implicit $vcc S_BRANCH %bb.2 @@ -794,6 +863,229 @@ body: | S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %5, %4, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + S_ENDPGM 0 + +... + +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_local_split +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + bb.0: + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_local_split + ; CHECK: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 killed $vgpr0_vgpr1, $vgpr0_vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + ; No VGPRs available for %0 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... + +# Performs a split and inflate around the single instruction +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_instruction_split +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + bb.0: + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_instruction_split + ; CHECK: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: SI_SPILL_AV64_SAVE killed $vgpr0_vgpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr2_vgpr3 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5) + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 killed $vgpr2_vgpr3, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + ; No VGPRs available for %0 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... + +# Performs a split and inflate around the single instruction, non-tied case +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_instruction_split +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + bb.0: + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_instruction_split + ; CHECK: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: SI_SPILL_AV64_SAVE killed $vgpr0_vgpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vgpr18_vgpr19 = COPY killed renamable $agpr0_agpr1 + ; CHECK-NEXT: early-clobber renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr0_vgpr1, $vgpr0_vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + ; No VGPRs available for %0 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) @@ -802,9 +1094,11 @@ body: | ... -# Chain of 2 untied cases, but the use isn't in src2. +# This case does not fully use %0 after the MFMA. As a result, +# SplitKits insert a copy bundle for the subset of used lanes instead +# of a simple copy. --- -name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_non_src2 +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_instruction_split_partial_uses_only tracksRegLiveness: true machineFunctionInfo: isEntryFunction: true @@ -812,7 +1106,447 @@ machineFunctionInfo: occupancy: 10 sgprForEXECCopy: '$sgpr100_sgpr101' body: | - ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_non_src2 + bb.0: + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_instruction_split_partial_uses_only + ; CHECK: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: SI_SPILL_AV64_SAVE killed $vgpr0_vgpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $agpr0_agpr1 + ; CHECK-NEXT: renamable $vgpr2_vgpr3 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 killed $vgpr2_vgpr3, $vgpr2_vgpr3, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY renamable $vgpr0_vgpr1_vgpr2_vgpr3 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = COPY renamable $agpr0_agpr1_agpr2_agpr3 + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + ; No VGPRs available for %0 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + S_ENDPGM 0 + +... + +# Untied version of previous. This case does not fully use %4 after +# the MFMA. As a result, SplitKits insert a copy bundle for the subset +# of used lanes instead of a simple copy, +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_instruction_split_partial_uses_only +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + bb.0: + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_instruction_split_partial_uses_only + ; CHECK: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: SI_SPILL_AV64_SAVE killed $vgpr0_vgpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vgpr18_vgpr19 = COPY killed renamable $agpr0_agpr1 + ; CHECK-NEXT: early-clobber renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr0_vgpr1, $vgpr0_vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY renamable $vgpr2_vgpr3_vgpr4_vgpr5 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = COPY renamable $agpr0_agpr1_agpr2_agpr3 + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + ; No VGPRs available for %4 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + S_ENDPGM 0 + +... + +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_same_subreg +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_same_subreg + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr10 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: renamable $vgpr11 = COPY renamable $vgpr10 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $vcc, $vgpr0_vgpr1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1) + ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1) + ; CHECK-NEXT: renamable $vgpr10_vgpr11_vgpr12_vgpr13 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1) + ; CHECK-NEXT: renamable $vgpr14_vgpr15_vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1) + ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000FFFFFFFF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + bb.0: + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_1024_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_1024_align2 = COPY %0.sub8 + + bb.1: + liveins: $vcc + + %0.sub0_sub1_sub2_sub3:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1) + %0.sub4_sub5_sub6_sub7:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1) + %0.sub8_sub9_sub10_sub11:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1) + %0.sub12_sub13_sub14_sub15:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1) + %0.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:vreg_1024_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_VCCNZ %bb.1, implicit $vcc + S_BRANCH %bb.2 + + bb.2: + ; No VGPRs available for %0 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... + +--- +name: chained_mfma_dst_user_is_vgpr +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + ; CHECK-LABEL: name: chained_mfma_dst_user_is_vgpr + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr16_vgpr17 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $vcc, $vgpr16_vgpr17 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 27983881 /* reguse:VReg_512_Align2 */, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000FFFFFFFF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + bb.0: + successors: %bb.1(0x80000000) + + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + + bb.1: + successors: %bb.1(0x40000000), %bb.2(0x40000000) + liveins: $vcc + + undef %2.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + early-clobber %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %2, 0, 0, 0, implicit $mode, implicit $exec + early-clobber %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 27983881 /* reguse:VReg_512_Align2 */, %4 + S_CBRANCH_VCCNZ %bb.1, implicit $vcc + S_BRANCH %bb.2 + + bb.2: + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %6:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... + +# TODO: In this trivial case, the single copy required is cheaper than +# the tuple copy. +--- +name: chained_mfma_dst_user_is_vgpr_small_subreg +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + ; CHECK-LABEL: name: chained_mfma_dst_user_is_vgpr_small_subreg + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr16_vgpr17 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $vcc, $vgpr16_vgpr17 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: dead renamable $vgpr0 = nofpexcept V_CVT_F16_F32_e32 killed $vgpr0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000FFFFFFFF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + bb.0: + successors: %bb.1(0x80000000) + + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + + bb.1: + successors: %bb.1(0x40000000), %bb.2(0x40000000) + liveins: $vcc + + undef %2.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + early-clobber %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %2, 0, 0, 0, implicit $mode, implicit $exec + early-clobber %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + %5:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 %4.sub0, implicit $mode, implicit $exec + S_CBRANCH_VCCNZ %bb.1, implicit $vcc + S_BRANCH %bb.2 + + bb.2: + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %6:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... + +# Transitive user of the register is an MFMA with non-register src2 +--- +name: chained_mfma_dst_user_has_imm_src2 +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + ; CHECK-LABEL: name: chained_mfma_dst_user_has_imm_src2 ; CHECK: bb.0: ; CHECK-NEXT: successors: %bb.1(0x80000000) ; CHECK-NEXT: {{ $}} @@ -830,7 +1564,8 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec - ; CHECK-NEXT: early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr4_vgpr5, $vgpr8_vgpr9, undef $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr20_vgpr21_vgpr22_vgpr23 = V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr20_vgpr21, $vgpr18_vgpr19, 0, 0, 0, 0, implicit $mode, implicit $exec ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc ; CHECK-NEXT: S_BRANCH %bb.2 ; CHECK-NEXT: {{ $}} @@ -853,6 +1588,8 @@ body: | ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) ; CHECK-NEXT: S_ENDPGM 0 bb.0: + successors: %bb.1(0x80000000) + S_NOP 0, implicit-def $agpr0 renamable $sgpr0 = S_MOV_B32 0 undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec @@ -862,16 +1599,104 @@ body: | %0.sub9:vreg_512_align2 = COPY %0.sub8 bb.1: + successors: %bb.1(0x40000000), %bb.2(0x40000000) liveins: $vcc - undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) - %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec - %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %3.sub4_sub5, %3.sub8_sub9, undef %6:vreg_512_align2, 0, 0, 0, implicit $mode, implicit $exec + undef %2.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + early-clobber %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %2, 0, 0, 0, implicit $mode, implicit $exec + early-clobber %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + %4.sub0_sub1_sub2_sub3:vreg_512_align2 = V_MFMA_F32_4X4X4F16_vgprcd_e64 %4.sub0_sub1, %1, 0, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_VCCNZ %bb.1, implicit $vcc S_BRANCH %bb.2 bb.2: - ; No VGPRs available for %0 or %4 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %6:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %6, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %6, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %6, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %6, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... + +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_has_untied_user +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_has_untied_user + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr16_vgpr17 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $vcc, $vgpr16_vgpr17 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000FFFFFFFF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + bb.0: + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + + bb.1: + liveins: $vcc + + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_VCCNZ %bb.1, implicit $vcc + S_BRANCH %bb.2 + + bb.2: + ; No VGPRs available for %0 S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 @@ -889,10 +1714,8 @@ body: | ... -# Chain of 2 untied cases, but the second mfma is a different size and -# uses a subregister. --- -name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_subreg +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_has_untied_user_with_vgpr_use tracksRegLiveness: true machineFunctionInfo: isEntryFunction: true @@ -900,7 +1723,7 @@ machineFunctionInfo: occupancy: 10 sgprForEXECCopy: '$sgpr100_sgpr101' body: | - ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_subreg + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_has_untied_user_with_vgpr_use ; CHECK: bb.0: ; CHECK-NEXT: successors: %bb.1(0x80000000) ; CHECK-NEXT: {{ $}} @@ -908,26 +1731,27 @@ body: | ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 - ; CHECK-NEXT: renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vgpr16_vgpr17 = COPY killed renamable $sgpr0_sgpr1 ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) - ; CHECK-NEXT: liveins: $vcc, $vgpr18_vgpr19 + ; CHECK-NEXT: liveins: $vcc, $vgpr16_vgpr17 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) - ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec - ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr2_vgpr3_vgpr4_vgpr5, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 27983881 /* reguse:VReg_512_Align2 */, renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc ; CHECK-NEXT: S_BRANCH %bb.2 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: - ; CHECK-NEXT: liveins: $vgpr0_vgpr1_vgpr2_vgpr3 + ; CHECK-NEXT: liveins: $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000FFFFFFFF ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 @@ -935,7 +1759,10 @@ body: | ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec - ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) ; CHECK-NEXT: S_ENDPGM 0 bb.0: S_NOP 0, implicit-def $agpr0 @@ -949,14 +1776,15 @@ body: | bb.1: liveins: $vcc - undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) - %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec - %4:vreg_128_align2 = V_MFMA_F32_16X16X16F16_vgprcd_e64 %1, %1, %3.sub2_sub3_sub4_sub5, 0, 0, 0, implicit $mode, implicit $exec + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 27983881 /* reguse:VReg_512_Align2 */, %4 S_CBRANCH_VCCNZ %bb.1, implicit $vcc S_BRANCH %bb.2 bb.2: - ; No VGPRs available for %0 or %4 + ; No VGPRs available for %0 S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 @@ -966,13 +1794,16 @@ body: | S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec - GLOBAL_STORE_DWORDX4_SADDR %5, %4, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) S_ENDPGM 0 ... --- -name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_same_subreg +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_has_tied_user tracksRegLiveness: true machineFunctionInfo: isEntryFunction: true @@ -980,32 +1811,115 @@ machineFunctionInfo: occupancy: 10 sgprForEXECCopy: '$sgpr100_sgpr101' body: | - ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_same_subreg + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_has_tied_user ; CHECK: bb.0: ; CHECK-NEXT: successors: %bb.1(0x80000000) ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0 ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 - ; CHECK-NEXT: renamable $vgpr10 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 + ; CHECK-NEXT: renamable $vgpr2_vgpr3 = COPY killed renamable $sgpr0_sgpr1 + ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $vcc, $vgpr2_vgpr3 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1) + ; CHECK-NEXT: early-clobber renamable $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, $vgpr2_vgpr3, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr2_vgpr3, $vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19:0x00000000FFFFFFFF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + bb.0: + S_NOP 0, implicit-def $agpr0 + renamable $sgpr0 = S_MOV_B32 0 + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec + renamable $sgpr1 = COPY renamable $sgpr0 + %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 + renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc + %0.sub9:vreg_512_align2 = COPY %0.sub8 + + bb.1: + liveins: $vcc + + undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1) + %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec + %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %4, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_VCCNZ %bb.1, implicit $vcc + S_BRANCH %bb.2 + + bb.2: + ; No VGPRs available for %0 + S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 + S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 + S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 + S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 + S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 + S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 + S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 + %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1) + GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... + +# Non-mac variant, src2 is an immediate. +--- +name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_imm_src2 +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_imm_src2 + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0 + ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0 + ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0 ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1 ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc - ; CHECK-NEXT: renamable $vgpr11 = COPY renamable $vgpr10 + ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) ; CHECK-NEXT: liveins: $vcc, $vgpr0_vgpr1 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1) - ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1) - ; CHECK-NEXT: renamable $vgpr10_vgpr11_vgpr12_vgpr13 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1) - ; CHECK-NEXT: renamable $vgpr14_vgpr15_vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1) - ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, 0, 0, 0, 0, implicit $mode, implicit $exec ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc ; CHECK-NEXT: S_BRANCH %bb.2 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: - ; CHECK-NEXT: liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000FFFFFFFF + ; CHECK-NEXT: liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17:0x00000000FFFFFFFF ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 @@ -1025,20 +1939,16 @@ body: | bb.0: S_NOP 0, implicit-def $agpr0 renamable $sgpr0 = S_MOV_B32 0 - undef %0.sub8:vreg_1024_align2 = V_MOV_B32_e32 0, implicit $exec + undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec renamable $sgpr1 = COPY renamable $sgpr0 %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1 renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc - %0.sub9:vreg_1024_align2 = COPY %0.sub8 + %0.sub9:vreg_512_align2 = COPY %0.sub8 bb.1: liveins: $vcc - %0.sub0_sub1_sub2_sub3:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1) - %0.sub4_sub5_sub6_sub7:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1) - %0.sub8_sub9_sub10_sub11:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1) - %0.sub12_sub13_sub14_sub15:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1) - %0.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:vreg_1024_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, 0, 0, 0, implicit $mode, implicit $exec + %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, 0, 0, 0, 0, implicit $mode, implicit $exec S_CBRANCH_VCCNZ %bb.1, implicit $vcc S_BRANCH %bb.2 diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll index 0c6339e..0b43ff2 100644 --- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll +++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; RUN: llc -mcpu=gfx90a < %s | FileCheck %s +; RUN: llc -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck %s target triple = "amdgcn-amd-amdhsa" @@ -7,7 +7,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma(ptr addrsp ; CHECK-LABEL: test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma: ; CHECK: ; %bb.0: ; %bb ; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; CHECK-NEXT: v_lshlrev_b32_e32 v0, 7, v0 +; CHECK-NEXT: v_mov_b32_e32 v32, 1.0 +; CHECK-NEXT: v_mov_b32_e32 v33, 2.0 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: global_load_dwordx4 v[28:31], v0, s[0:1] offset:112 ; CHECK-NEXT: global_load_dwordx4 v[24:27], v0, s[0:1] offset:96 @@ -18,117 +21,58 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma(ptr addrsp ; CHECK-NEXT: global_load_dwordx4 v[4:7], v0, s[0:1] offset:16 ; CHECK-NEXT: s_nop 0 ; CHECK-NEXT: global_load_dwordx4 v[0:3], v0, s[0:1] +; CHECK-NEXT: v_accvgpr_write_b32 a0, 1.0 +; CHECK-NEXT: v_accvgpr_write_b32 a1, 2.0 ; CHECK-NEXT: s_waitcnt vmcnt(0) -; CHECK-NEXT: v_accvgpr_write_b32 a0, v0 -; CHECK-NEXT: v_accvgpr_write_b32 a1, v1 -; CHECK-NEXT: v_accvgpr_write_b32 a2, v2 -; CHECK-NEXT: v_accvgpr_write_b32 a3, v3 -; CHECK-NEXT: v_accvgpr_write_b32 a4, v4 -; CHECK-NEXT: v_accvgpr_write_b32 a5, v5 -; CHECK-NEXT: v_accvgpr_write_b32 a6, v6 -; CHECK-NEXT: v_accvgpr_write_b32 a7, v7 -; CHECK-NEXT: v_accvgpr_write_b32 a8, v8 -; CHECK-NEXT: v_accvgpr_write_b32 a9, v9 -; CHECK-NEXT: v_accvgpr_write_b32 a10, v10 -; CHECK-NEXT: v_accvgpr_write_b32 a11, v11 -; CHECK-NEXT: v_accvgpr_write_b32 a12, v12 -; CHECK-NEXT: v_accvgpr_write_b32 a13, v13 -; CHECK-NEXT: v_accvgpr_write_b32 a14, v14 -; CHECK-NEXT: v_accvgpr_write_b32 a15, v15 -; CHECK-NEXT: v_accvgpr_write_b32 a16, v16 -; CHECK-NEXT: v_accvgpr_write_b32 a17, v17 -; CHECK-NEXT: v_accvgpr_write_b32 a18, v18 -; CHECK-NEXT: v_accvgpr_write_b32 a19, v19 -; CHECK-NEXT: v_accvgpr_write_b32 a20, v20 -; CHECK-NEXT: v_accvgpr_write_b32 a21, v21 -; CHECK-NEXT: v_accvgpr_write_b32 a22, v22 -; CHECK-NEXT: v_accvgpr_write_b32 a23, v23 -; CHECK-NEXT: v_accvgpr_write_b32 a24, v24 -; CHECK-NEXT: v_accvgpr_write_b32 a25, v25 -; CHECK-NEXT: v_accvgpr_write_b32 a26, v26 -; CHECK-NEXT: v_accvgpr_write_b32 a27, v27 -; CHECK-NEXT: v_accvgpr_write_b32 a28, v28 -; CHECK-NEXT: v_accvgpr_write_b32 a29, v29 -; CHECK-NEXT: v_accvgpr_write_b32 a30, v30 -; CHECK-NEXT: v_accvgpr_write_b32 a31, v31 -; CHECK-NEXT: v_mov_b32_e32 v0, 1.0 -; CHECK-NEXT: v_mov_b32_e32 v1, 2.0 -; CHECK-NEXT: s_nop 1 -; CHECK-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] -; CHECK-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v0, v1, a[0:31] +; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] +; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[32:63], a0, a1, v[0:31] ; CHECK-NEXT: s_nop 7 ; CHECK-NEXT: s_nop 7 -; CHECK-NEXT: s_nop 2 -; CHECK-NEXT: v_accvgpr_read_b32 v4, a59 -; CHECK-NEXT: v_accvgpr_read_b32 v5, a58 -; CHECK-NEXT: v_accvgpr_read_b32 v6, a57 -; CHECK-NEXT: v_accvgpr_read_b32 v7, a56 -; CHECK-NEXT: v_accvgpr_read_b32 v8, a55 -; CHECK-NEXT: v_accvgpr_read_b32 v9, a54 -; CHECK-NEXT: v_accvgpr_read_b32 v10, a53 -; CHECK-NEXT: v_accvgpr_read_b32 v11, a52 -; CHECK-NEXT: v_accvgpr_read_b32 v12, a51 -; CHECK-NEXT: v_accvgpr_read_b32 v13, a50 -; CHECK-NEXT: v_accvgpr_read_b32 v14, a49 -; CHECK-NEXT: v_accvgpr_read_b32 v15, a48 -; CHECK-NEXT: v_accvgpr_read_b32 v16, a47 -; CHECK-NEXT: v_accvgpr_read_b32 v17, a46 -; CHECK-NEXT: v_accvgpr_read_b32 v18, a45 -; CHECK-NEXT: v_accvgpr_read_b32 v19, a44 -; CHECK-NEXT: v_accvgpr_read_b32 v20, a43 -; CHECK-NEXT: v_accvgpr_read_b32 v21, a42 -; CHECK-NEXT: v_accvgpr_read_b32 v22, a41 -; CHECK-NEXT: v_accvgpr_read_b32 v23, a40 -; CHECK-NEXT: v_accvgpr_read_b32 v24, a39 -; CHECK-NEXT: v_accvgpr_read_b32 v25, a38 -; CHECK-NEXT: v_accvgpr_read_b32 v26, a37 -; CHECK-NEXT: v_accvgpr_read_b32 v27, a36 -; CHECK-NEXT: v_accvgpr_read_b32 v28, a35 -; CHECK-NEXT: v_accvgpr_read_b32 v29, a34 -; CHECK-NEXT: v_accvgpr_mov_b32 a2, a32 -; CHECK-NEXT: v_accvgpr_mov_b32 a3, a33 -; CHECK-NEXT: v_accvgpr_write_b32 a4, v29 -; CHECK-NEXT: v_accvgpr_write_b32 a5, v28 -; CHECK-NEXT: v_accvgpr_write_b32 a6, v27 -; CHECK-NEXT: v_accvgpr_write_b32 a7, v26 -; CHECK-NEXT: v_accvgpr_write_b32 a8, v25 -; CHECK-NEXT: v_accvgpr_write_b32 a9, v24 -; CHECK-NEXT: v_accvgpr_write_b32 a10, v23 -; CHECK-NEXT: v_accvgpr_write_b32 a11, v22 -; CHECK-NEXT: v_accvgpr_write_b32 a12, v21 -; CHECK-NEXT: v_accvgpr_write_b32 a13, v20 -; CHECK-NEXT: v_accvgpr_write_b32 a14, v19 -; CHECK-NEXT: v_accvgpr_write_b32 a15, v18 -; CHECK-NEXT: v_accvgpr_write_b32 a16, v17 -; CHECK-NEXT: v_accvgpr_write_b32 a17, v16 -; CHECK-NEXT: v_accvgpr_write_b32 a18, v15 -; CHECK-NEXT: v_accvgpr_write_b32 a19, v14 -; CHECK-NEXT: v_accvgpr_write_b32 a20, v13 -; CHECK-NEXT: v_accvgpr_write_b32 a21, v12 -; CHECK-NEXT: v_accvgpr_write_b32 a22, v11 -; CHECK-NEXT: v_accvgpr_write_b32 a23, v10 -; CHECK-NEXT: v_accvgpr_write_b32 a24, v9 -; CHECK-NEXT: v_accvgpr_write_b32 a25, v8 -; CHECK-NEXT: v_accvgpr_write_b32 a26, v7 -; CHECK-NEXT: v_accvgpr_write_b32 a27, v6 -; CHECK-NEXT: v_accvgpr_write_b32 a28, v5 -; CHECK-NEXT: v_accvgpr_write_b32 a29, v4 -; CHECK-NEXT: v_accvgpr_mov_b32 a30, a60 -; CHECK-NEXT: v_accvgpr_mov_b32 a31, a61 ; CHECK-NEXT: s_nop 1 -; CHECK-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] -; CHECK-NEXT: v_mov_b32_e32 v0, 0 +; CHECK-NEXT: v_mov_b32_e32 v2, v32 +; CHECK-NEXT: v_mov_b32_e32 v3, v33 +; CHECK-NEXT: v_mov_b32_e32 v4, v34 +; CHECK-NEXT: v_mov_b32_e32 v5, v35 +; CHECK-NEXT: v_mov_b32_e32 v6, v36 +; CHECK-NEXT: v_mov_b32_e32 v7, v37 +; CHECK-NEXT: v_mov_b32_e32 v8, v38 +; CHECK-NEXT: v_mov_b32_e32 v9, v39 +; CHECK-NEXT: v_mov_b32_e32 v10, v40 +; CHECK-NEXT: v_mov_b32_e32 v11, v41 +; CHECK-NEXT: v_mov_b32_e32 v12, v42 +; CHECK-NEXT: v_mov_b32_e32 v13, v43 +; CHECK-NEXT: v_mov_b32_e32 v14, v44 +; CHECK-NEXT: v_mov_b32_e32 v15, v45 +; CHECK-NEXT: v_mov_b32_e32 v16, v46 +; CHECK-NEXT: v_mov_b32_e32 v17, v47 +; CHECK-NEXT: v_mov_b32_e32 v18, v48 +; CHECK-NEXT: v_mov_b32_e32 v19, v49 +; CHECK-NEXT: v_mov_b32_e32 v20, v50 +; CHECK-NEXT: v_mov_b32_e32 v21, v51 +; CHECK-NEXT: v_mov_b32_e32 v22, v52 +; CHECK-NEXT: v_mov_b32_e32 v23, v53 +; CHECK-NEXT: v_mov_b32_e32 v24, v54 +; CHECK-NEXT: v_mov_b32_e32 v25, v55 +; CHECK-NEXT: v_mov_b32_e32 v26, v56 +; CHECK-NEXT: v_mov_b32_e32 v27, v57 +; CHECK-NEXT: v_mov_b32_e32 v28, v58 +; CHECK-NEXT: v_mov_b32_e32 v29, v59 +; CHECK-NEXT: v_mov_b32_e32 v30, v60 +; CHECK-NEXT: v_mov_b32_e32 v31, v61 +; CHECK-NEXT: v_mov_b32_e32 v32, 0 +; CHECK-NEXT: s_nop 0 +; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], a0, a1, v[0:31] ; CHECK-NEXT: s_nop 7 ; CHECK-NEXT: s_nop 7 ; CHECK-NEXT: s_nop 1 -; CHECK-NEXT: global_store_dwordx4 v0, a[24:27], s[0:1] offset:96 -; CHECK-NEXT: global_store_dwordx4 v0, a[28:31], s[0:1] offset:112 -; CHECK-NEXT: global_store_dwordx4 v0, a[16:19], s[0:1] offset:64 -; CHECK-NEXT: global_store_dwordx4 v0, a[20:23], s[0:1] offset:80 -; CHECK-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 -; CHECK-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 -; CHECK-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] -; CHECK-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; CHECK-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96 +; CHECK-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112 +; CHECK-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64 +; CHECK-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80 +; CHECK-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 +; CHECK-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 +; CHECK-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] +; CHECK-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 ; CHECK-NEXT: s_endpgm bb: %id = call i32 @llvm.amdgcn.workitem.id.x() @@ -146,35 +90,36 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_noshuffle( ; CHECK-LABEL: test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_noshuffle: ; CHECK: ; %bb.0: ; %bb ; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; CHECK-NEXT: v_lshlrev_b32_e32 v0, 7, v0 -; CHECK-NEXT: v_mov_b32_e32 v1, 2.0 +; CHECK-NEXT: v_mov_b32_e32 v32, 1.0 +; CHECK-NEXT: v_mov_b32_e32 v33, 2.0 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) -; CHECK-NEXT: global_load_dwordx4 a[28:31], v0, s[0:1] offset:112 -; CHECK-NEXT: global_load_dwordx4 a[24:27], v0, s[0:1] offset:96 -; CHECK-NEXT: global_load_dwordx4 a[20:23], v0, s[0:1] offset:80 -; CHECK-NEXT: global_load_dwordx4 a[16:19], v0, s[0:1] offset:64 -; CHECK-NEXT: global_load_dwordx4 a[12:15], v0, s[0:1] offset:48 -; CHECK-NEXT: global_load_dwordx4 a[8:11], v0, s[0:1] offset:32 -; CHECK-NEXT: global_load_dwordx4 a[4:7], v0, s[0:1] offset:16 -; CHECK-NEXT: global_load_dwordx4 a[0:3], v0, s[0:1] -; CHECK-NEXT: v_mov_b32_e32 v0, 1.0 -; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: global_load_dwordx4 v[28:31], v0, s[0:1] offset:112 +; CHECK-NEXT: global_load_dwordx4 v[24:27], v0, s[0:1] offset:96 +; CHECK-NEXT: global_load_dwordx4 v[20:23], v0, s[0:1] offset:80 +; CHECK-NEXT: global_load_dwordx4 v[16:19], v0, s[0:1] offset:64 +; CHECK-NEXT: global_load_dwordx4 v[12:15], v0, s[0:1] offset:48 +; CHECK-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1] offset:32 +; CHECK-NEXT: global_load_dwordx4 v[4:7], v0, s[0:1] offset:16 ; CHECK-NEXT: s_nop 0 -; CHECK-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] -; CHECK-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] -; CHECK-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] -; CHECK-NEXT: v_mov_b32_e32 v0, 0 +; CHECK-NEXT: global_load_dwordx4 v[0:3], v0, s[0:1] +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] +; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] +; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] +; CHECK-NEXT: v_mov_b32_e32 v32, 0 ; CHECK-NEXT: s_nop 7 ; CHECK-NEXT: s_nop 7 -; CHECK-NEXT: s_nop 1 -; CHECK-NEXT: global_store_dwordx4 v0, a[24:27], s[0:1] offset:96 -; CHECK-NEXT: global_store_dwordx4 v0, a[28:31], s[0:1] offset:112 -; CHECK-NEXT: global_store_dwordx4 v0, a[16:19], s[0:1] offset:64 -; CHECK-NEXT: global_store_dwordx4 v0, a[20:23], s[0:1] offset:80 -; CHECK-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 -; CHECK-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 -; CHECK-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] -; CHECK-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; CHECK-NEXT: s_nop 0 +; CHECK-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96 +; CHECK-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112 +; CHECK-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64 +; CHECK-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80 +; CHECK-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 +; CHECK-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 +; CHECK-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] +; CHECK-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 ; CHECK-NEXT: s_endpgm bb: %id = call i32 @llvm.amdgcn.workitem.id.x() @@ -187,9 +132,77 @@ bb: ret void } +define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_imm0_src2(ptr addrspace(1) %arg) #0 { +; CHECK-LABEL: test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_imm0_src2: +; CHECK: ; %bb.0: ; %bb +; CHECK-NEXT: v_mov_b32_e32 v32, 1.0 +; CHECK-NEXT: v_mov_b32_e32 v33, 2.0 +; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; CHECK-NEXT: s_nop 0 +; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, 0 +; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] +; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] +; CHECK-NEXT: v_mov_b32_e32 v32, 0 +; CHECK-NEXT: s_waitcnt lgkmcnt(0) +; CHECK-NEXT: s_nop 7 +; CHECK-NEXT: s_nop 7 +; CHECK-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112 +; CHECK-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96 +; CHECK-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80 +; CHECK-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64 +; CHECK-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 +; CHECK-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 +; CHECK-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 +; CHECK-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] +; CHECK-NEXT: s_endpgm +bb: + %id = call i32 @llvm.amdgcn.workitem.id.x() + %gep = getelementptr <32 x float>, ptr addrspace(1) %arg, i32 %id + %in.1 = load <32 x float>, ptr addrspace(1) %gep, align 128 + %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> zeroinitializer, i32 0, i32 0, i32 0) + %mai.2 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %mai.1, i32 0, i32 0, i32 0) + %mai.3 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %mai.2, i32 0, i32 0, i32 0) + store <32 x float> %mai.3, ptr addrspace(1) %arg, align 128 + ret void +} + +define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_imm1_src2(ptr addrspace(1) %arg) #0 { +; CHECK-LABEL: test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_imm1_src2: +; CHECK: ; %bb.0: ; %bb +; CHECK-NEXT: v_mov_b32_e32 v32, 1.0 +; CHECK-NEXT: v_mov_b32_e32 v33, 2.0 +; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; CHECK-NEXT: s_nop 0 +; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, 1.0 +; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] +; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] +; CHECK-NEXT: v_mov_b32_e32 v32, 0 +; CHECK-NEXT: s_waitcnt lgkmcnt(0) +; CHECK-NEXT: s_nop 7 +; CHECK-NEXT: s_nop 7 +; CHECK-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112 +; CHECK-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96 +; CHECK-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80 +; CHECK-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64 +; CHECK-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 +; CHECK-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 +; CHECK-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 +; CHECK-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] +; CHECK-NEXT: s_endpgm +bb: + %id = call i32 @llvm.amdgcn.workitem.id.x() + %gep = getelementptr <32 x float>, ptr addrspace(1) %arg, i32 %id + %in.1 = load <32 x float>, ptr addrspace(1) %gep, align 128 + %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> splat (float 1.0), i32 0, i32 0, i32 0) + %mai.2 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %mai.1, i32 0, i32 0, i32 0) + %mai.3 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %mai.2, i32 0, i32 0, i32 0) + store <32 x float> %mai.3, ptr addrspace(1) %arg, align 128 + ret void +} + declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32 immarg, i32 immarg, i32 immarg) #1 declare noundef i32 @llvm.amdgcn.workitem.id.x() #2 -attributes #0 = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,4" } +attributes #0 = { nounwind "amdgpu-flat-work-group-size"="1,256" "amdgpu-waves-per-eu"="4,4" } attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) } attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } diff --git a/llvm/test/CodeGen/AVR/cmp.ll b/llvm/test/CodeGen/AVR/cmp.ll index efc9b8d..c932bda1 100644 --- a/llvm/test/CodeGen/AVR/cmp.ll +++ b/llvm/test/CodeGen/AVR/cmp.ll @@ -298,3 +298,18 @@ define i16 @cmp_i16_gt_1023(i16 %0) { %3 = zext i1 %2 to i16 ret i16 %3 } + +define void @cmp_issue152097(i16 %a) addrspace(1) { +; See: https://github.com/llvm/llvm-project/issues/152097 +; CHECK-LABEL: cmp_issue152097 +; CHECK: ldi r18, -1 +; CHECK-NEXT: cpi r24, -2 +; CHECK-NEXT: cpc r25, r18 +; CHECK-NEXT: ret + %cmp = icmp ugt i16 -2, %a + br i1 %cmp, label %if.then, label %if.else +if.then: + ret void +if.else: + ret void +} diff --git a/llvm/test/CodeGen/NVPTX/prefetch-inferas-test.ll b/llvm/test/CodeGen/NVPTX/prefetch-inferas-test.ll new file mode 100644 index 0000000..822a75e --- /dev/null +++ b/llvm/test/CodeGen/NVPTX/prefetch-inferas-test.ll @@ -0,0 +1,78 @@ +; RUN: opt < %s -S -passes=infer-address-spaces | FileCheck %s --check-prefix=INFER
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80 | FileCheck %s --check-prefix=PTX
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80 | %ptxas-verify %}
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"
+target triple = "nvptx64-unknown-unknown"
+
+@constant_tensormap = addrspace(4) global [64 x i8] zeroinitializer, align 64
+
+; Inference from const address space
+define void @test_infer_const_from_cast() {
+; INFER-LABEL: @test_infer_const_from_cast
+; INFER: call void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) @constant_tensormap)
+; BOTH: call void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) @constant_tensormap)
+; PTX-LABEL: .visible .func test_infer_const_from_cast(
+; PTX: mov.b64 %rd{{[0-9]+}}, constant_tensormap;
+; PTX: cvta.const.u64 %rd{{[0-9]+}}, %rd{{[0-9]+}};
+; PTX: prefetch.tensormap [%rd{{[0-9]+}}];
+entry:
+ %casted = addrspacecast ptr addrspace(4) @constant_tensormap to ptr
+ call void @llvm.nvvm.prefetch.tensormap.p0(ptr %casted)
+ ret void
+}
+
+; Cast from Const space to Generic
+define void @test_const_to_generic_cast(ptr addrspace(4) %const_ptr) {
+; INFER-LABEL: @test_const_to_generic_cast
+; INFER: call void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) %const_ptr)
+; PTX-LABEL: .visible .func test_const_to_generic_cast(
+; PTX: prefetch.const.tensormap [%rd{{[0-9]+}}];
+entry:
+ %cast = addrspacecast ptr addrspace(4) %const_ptr to ptr
+ call void @llvm.nvvm.prefetch.tensormap.p0(ptr %cast)
+ ret void
+}
+
+; No inference possible
+define void @test_no_inference_possible(ptr %generic_ptr) {
+; INFER-LABEL: @test_no_inference_possible
+; INFER: call void @llvm.nvvm.prefetch.tensormap.p0(ptr %generic_ptr)
+; PTX-LABEL: .visible .func test_no_inference_possible(
+; PTX: prefetch.tensormap [%rd{{[0-9]+}}];
+entry:
+ call void @llvm.nvvm.prefetch.tensormap.p0(ptr %generic_ptr)
+ ret void
+}
+
+; Cast from Parameter space to Generic
+define void @test_param_to_generic_cast(ptr addrspace(101) %param_ptr) {
+; INFER-LABEL: @test_param_to_generic_cast
+; INFER: call void @llvm.nvvm.prefetch.tensormap.p101(ptr addrspace(101) %param_ptr)
+; PTX-LABEL: .visible .func test_param_to_generic_cast(
+; PTX: prefetch.param.tensormap [%rd{{[0-9]+}}];
+entry:
+ %cast = addrspacecast ptr addrspace(101) %param_ptr to ptr
+ call void @llvm.nvvm.prefetch.tensormap.p0(ptr %cast)
+ ret void
+}
+
+; Multiple casts in sequence
+define void @test_infer_through_multiple_casts() {
+; INFER-LABEL: @test_infer_through_multiple_casts
+; INFER: call void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) @constant_tensormap)
+; PTX-LABEL: .visible .func test_infer_through_multiple_casts(
+; PTX: mov.b64 %rd{{[0-9]+}}, constant_tensormap;
+; PTX: cvta.const.u64 %rd{{[0-9]+}}, %rd{{[0-9]+}};
+; PTX: prefetch.tensormap [%rd{{[0-9]+}}];
+entry:
+ %cast1 = addrspacecast ptr addrspace(4) @constant_tensormap to ptr
+ %cast2 = addrspacecast ptr %cast1 to ptr addrspace(4)
+ %cast3 = addrspacecast ptr addrspace(4) %cast2 to ptr
+ call void @llvm.nvvm.prefetch.tensormap(ptr %cast3)
+ ret void
+}
+
+declare void @llvm.nvvm.prefetch.tensormap.p0(ptr)
+declare void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4))
+declare void @llvm.nvvm.prefetch.tensormap.p101(ptr addrspace(101))
diff --git a/llvm/test/CodeGen/NVPTX/prefetch.ll b/llvm/test/CodeGen/NVPTX/prefetch.ll index a64e4fe..862e26d 100644 --- a/llvm/test/CodeGen/NVPTX/prefetch.ll +++ b/llvm/test/CodeGen/NVPTX/prefetch.ll @@ -12,6 +12,10 @@ declare void @llvm.nvvm.prefetch.local.L2(ptr addrspace(5) %local_ptr) declare void @llvm.nvvm.prefetch.L1(ptr %ptr)
declare void @llvm.nvvm.prefetch.L2(ptr %ptr)
+declare void @llvm.nvvm.prefetch.tensormap.p0(ptr %ptr)
+declare void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) %const_ptr)
+declare void @llvm.nvvm.prefetch.tensormap.p101(ptr addrspace(101) %param_ptr)
+
declare void @llvm.nvvm.prefetch.global.L2.evict.normal(ptr addrspace(1) %global_ptr)
declare void @llvm.nvvm.prefetch.global.L2.evict.last(ptr addrspace(1) %global_ptr)
@@ -78,4 +82,43 @@ define void @prefetchu_l1(ptr %ptr) { ; CHECK-PTX64-NEXT: ret;
tail call void @llvm.nvvm.prefetchu.L1(ptr %ptr)
ret void
+}
+
+define void @prefetch_tensormap(ptr %ptr) {
+; CHECK-PTX64-LABEL: prefetch_tensormap(
+; CHECK-PTX64: {
+; CHECK-PTX64-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT: // %bb.0:
+; CHECK-PTX64-NEXT: ld.param.b64 %rd1, [prefetch_tensormap_param_0];
+; CHECK-PTX64-NEXT: prefetch.tensormap [%rd1];
+; CHECK-PTX64-NEXT: ret;
+ tail call void @llvm.nvvm.prefetch.tensormap.p0(ptr %ptr)
+ ret void
+}
+
+define void @prefetch_const_tensormap(ptr addrspace(4) %const_ptr) {
+; CHECK-PTX64-LABEL: prefetch_const_tensormap(
+; CHECK-PTX64: {
+; CHECK-PTX64-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT: // %bb.0:
+; CHECK-PTX64-NEXT: ld.param.b64 %rd1, [prefetch_const_tensormap_param_0];
+; CHECK-PTX64-NEXT: prefetch.const.tensormap [%rd1];
+; CHECK-PTX64-NEXT: ret;
+ tail call void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) %const_ptr)
+ ret void
+}
+
+define void @prefetch_param_tensormap(ptr addrspace(101) %param_ptr) {
+; CHECK-PTX64-LABEL: prefetch_param_tensormap(
+; CHECK-PTX64: {
+; CHECK-PTX64-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT: // %bb.0:
+; CHECK-PTX64-NEXT: ld.param.b64 %rd1, [prefetch_param_tensormap_param_0];
+; CHECK-PTX64-NEXT: prefetch.param.tensormap [%rd1];
+; CHECK-PTX64-NEXT: ret;
+ tail call void @llvm.nvvm.prefetch.tensormap.p101(ptr addrspace(101) %param_ptr)
+ ret void
}
\ No newline at end of file diff --git a/llvm/test/CodeGen/RISCV/half-convert.ll b/llvm/test/CodeGen/RISCV/half-convert.ll index facb544..0c152e6 100644 --- a/llvm/test/CodeGen/RISCV/half-convert.ll +++ b/llvm/test/CodeGen/RISCV/half-convert.ll @@ -2262,12 +2262,12 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; RV32IZHINX-NEXT: addi a2, a3, -1 ; RV32IZHINX-NEXT: .LBB10_4: # %start ; RV32IZHINX-NEXT: feq.s a3, s0, s0 -; RV32IZHINX-NEXT: neg a4, a1 -; RV32IZHINX-NEXT: neg a1, s1 +; RV32IZHINX-NEXT: neg a4, s1 +; RV32IZHINX-NEXT: neg a5, a1 ; RV32IZHINX-NEXT: neg a3, a3 -; RV32IZHINX-NEXT: and a0, a1, a0 +; RV32IZHINX-NEXT: and a0, a4, a0 ; RV32IZHINX-NEXT: and a1, a3, a2 -; RV32IZHINX-NEXT: or a0, a4, a0 +; RV32IZHINX-NEXT: or a0, a5, a0 ; RV32IZHINX-NEXT: and a0, a3, a0 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload @@ -2309,12 +2309,12 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; RV32IZDINXZHINX-NEXT: addi a2, a3, -1 ; RV32IZDINXZHINX-NEXT: .LBB10_4: # %start ; RV32IZDINXZHINX-NEXT: feq.s a3, s0, s0 -; RV32IZDINXZHINX-NEXT: neg a4, a1 -; RV32IZDINXZHINX-NEXT: neg a1, s1 +; RV32IZDINXZHINX-NEXT: neg a4, s1 +; RV32IZDINXZHINX-NEXT: neg a5, a1 ; RV32IZDINXZHINX-NEXT: neg a3, a3 -; RV32IZDINXZHINX-NEXT: and a0, a1, a0 +; RV32IZDINXZHINX-NEXT: and a0, a4, a0 ; RV32IZDINXZHINX-NEXT: and a1, a3, a2 -; RV32IZDINXZHINX-NEXT: or a0, a4, a0 +; RV32IZDINXZHINX-NEXT: or a0, a5, a0 ; RV32IZDINXZHINX-NEXT: and a0, a3, a0 ; RV32IZDINXZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZDINXZHINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload @@ -2653,12 +2653,12 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; CHECK32-IZHINXMIN-NEXT: addi a2, a3, -1 ; CHECK32-IZHINXMIN-NEXT: .LBB10_4: # %start ; CHECK32-IZHINXMIN-NEXT: feq.s a3, s0, s0 -; CHECK32-IZHINXMIN-NEXT: neg a4, a1 -; CHECK32-IZHINXMIN-NEXT: neg a1, s1 +; CHECK32-IZHINXMIN-NEXT: neg a4, s1 +; CHECK32-IZHINXMIN-NEXT: neg a5, a1 ; CHECK32-IZHINXMIN-NEXT: neg a3, a3 -; CHECK32-IZHINXMIN-NEXT: and a0, a1, a0 +; CHECK32-IZHINXMIN-NEXT: and a0, a4, a0 ; CHECK32-IZHINXMIN-NEXT: and a1, a3, a2 -; CHECK32-IZHINXMIN-NEXT: or a0, a4, a0 +; CHECK32-IZHINXMIN-NEXT: or a0, a5, a0 ; CHECK32-IZHINXMIN-NEXT: and a0, a3, a0 ; CHECK32-IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; CHECK32-IZHINXMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload @@ -2701,12 +2701,12 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; CHECK32-IZDINXZHINXMIN-NEXT: addi a2, a3, -1 ; CHECK32-IZDINXZHINXMIN-NEXT: .LBB10_4: # %start ; CHECK32-IZDINXZHINXMIN-NEXT: feq.s a3, s0, s0 -; CHECK32-IZDINXZHINXMIN-NEXT: neg a4, a1 -; CHECK32-IZDINXZHINXMIN-NEXT: neg a1, s1 +; CHECK32-IZDINXZHINXMIN-NEXT: neg a4, s1 +; CHECK32-IZDINXZHINXMIN-NEXT: neg a5, a1 ; CHECK32-IZDINXZHINXMIN-NEXT: neg a3, a3 -; CHECK32-IZDINXZHINXMIN-NEXT: and a0, a1, a0 +; CHECK32-IZDINXZHINXMIN-NEXT: and a0, a4, a0 ; CHECK32-IZDINXZHINXMIN-NEXT: and a1, a3, a2 -; CHECK32-IZDINXZHINXMIN-NEXT: or a0, a4, a0 +; CHECK32-IZDINXZHINXMIN-NEXT: or a0, a5, a0 ; CHECK32-IZDINXZHINXMIN-NEXT: and a0, a3, a0 ; CHECK32-IZDINXZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; CHECK32-IZDINXZHINXMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload @@ -2972,18 +2972,19 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; RV32IZHINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill -; RV32IZHINX-NEXT: fcvt.s.h a0, a0 -; RV32IZHINX-NEXT: lui a1, 391168 -; RV32IZHINX-NEXT: addi a1, a1, -1 -; RV32IZHINX-NEXT: fle.s a2, zero, a0 -; RV32IZHINX-NEXT: flt.s a1, a1, a0 -; RV32IZHINX-NEXT: neg s0, a1 -; RV32IZHINX-NEXT: neg s1, a2 +; RV32IZHINX-NEXT: fcvt.s.h s0, a0 +; RV32IZHINX-NEXT: fle.s a0, zero, s0 +; RV32IZHINX-NEXT: neg s1, a0 +; RV32IZHINX-NEXT: mv a0, s0 ; RV32IZHINX-NEXT: call __fixunssfdi ; RV32IZHINX-NEXT: and a0, s1, a0 +; RV32IZHINX-NEXT: lui a2, 391168 ; RV32IZHINX-NEXT: and a1, s1, a1 -; RV32IZHINX-NEXT: or a0, s0, a0 -; RV32IZHINX-NEXT: or a1, s0, a1 +; RV32IZHINX-NEXT: addi a2, a2, -1 +; RV32IZHINX-NEXT: flt.s a2, a2, s0 +; RV32IZHINX-NEXT: neg a2, a2 +; RV32IZHINX-NEXT: or a0, a2, a0 +; RV32IZHINX-NEXT: or a1, a2, a1 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload @@ -3005,18 +3006,19 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; RV32IZDINXZHINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZDINXZHINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill ; RV32IZDINXZHINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill -; RV32IZDINXZHINX-NEXT: fcvt.s.h a0, a0 -; RV32IZDINXZHINX-NEXT: lui a1, 391168 -; RV32IZDINXZHINX-NEXT: addi a1, a1, -1 -; RV32IZDINXZHINX-NEXT: fle.s a2, zero, a0 -; RV32IZDINXZHINX-NEXT: flt.s a1, a1, a0 -; RV32IZDINXZHINX-NEXT: neg s0, a1 -; RV32IZDINXZHINX-NEXT: neg s1, a2 +; RV32IZDINXZHINX-NEXT: fcvt.s.h s0, a0 +; RV32IZDINXZHINX-NEXT: fle.s a0, zero, s0 +; RV32IZDINXZHINX-NEXT: neg s1, a0 +; RV32IZDINXZHINX-NEXT: mv a0, s0 ; RV32IZDINXZHINX-NEXT: call __fixunssfdi ; RV32IZDINXZHINX-NEXT: and a0, s1, a0 +; RV32IZDINXZHINX-NEXT: lui a2, 391168 ; RV32IZDINXZHINX-NEXT: and a1, s1, a1 -; RV32IZDINXZHINX-NEXT: or a0, s0, a0 -; RV32IZDINXZHINX-NEXT: or a1, s0, a1 +; RV32IZDINXZHINX-NEXT: addi a2, a2, -1 +; RV32IZDINXZHINX-NEXT: flt.s a2, a2, s0 +; RV32IZDINXZHINX-NEXT: neg a2, a2 +; RV32IZDINXZHINX-NEXT: or a0, a2, a0 +; RV32IZDINXZHINX-NEXT: or a1, a2, a1 ; RV32IZDINXZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZDINXZHINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; RV32IZDINXZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload @@ -3217,18 +3219,19 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; CHECK32-IZHINXMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; CHECK32-IZHINXMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill ; CHECK32-IZHINXMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill -; CHECK32-IZHINXMIN-NEXT: fcvt.s.h a0, a0 -; CHECK32-IZHINXMIN-NEXT: lui a1, 391168 -; CHECK32-IZHINXMIN-NEXT: addi a1, a1, -1 -; CHECK32-IZHINXMIN-NEXT: fle.s a2, zero, a0 -; CHECK32-IZHINXMIN-NEXT: flt.s a1, a1, a0 -; CHECK32-IZHINXMIN-NEXT: neg s0, a1 -; CHECK32-IZHINXMIN-NEXT: neg s1, a2 +; CHECK32-IZHINXMIN-NEXT: fcvt.s.h s0, a0 +; CHECK32-IZHINXMIN-NEXT: fle.s a0, zero, s0 +; CHECK32-IZHINXMIN-NEXT: neg s1, a0 +; CHECK32-IZHINXMIN-NEXT: mv a0, s0 ; CHECK32-IZHINXMIN-NEXT: call __fixunssfdi ; CHECK32-IZHINXMIN-NEXT: and a0, s1, a0 +; CHECK32-IZHINXMIN-NEXT: lui a2, 391168 ; CHECK32-IZHINXMIN-NEXT: and a1, s1, a1 -; CHECK32-IZHINXMIN-NEXT: or a0, s0, a0 -; CHECK32-IZHINXMIN-NEXT: or a1, s0, a1 +; CHECK32-IZHINXMIN-NEXT: addi a2, a2, -1 +; CHECK32-IZHINXMIN-NEXT: flt.s a2, a2, s0 +; CHECK32-IZHINXMIN-NEXT: neg a2, a2 +; CHECK32-IZHINXMIN-NEXT: or a0, a2, a0 +; CHECK32-IZHINXMIN-NEXT: or a1, a2, a1 ; CHECK32-IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; CHECK32-IZHINXMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; CHECK32-IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload @@ -3251,18 +3254,19 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; CHECK32-IZDINXZHINXMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; CHECK32-IZDINXZHINXMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill ; CHECK32-IZDINXZHINXMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill -; CHECK32-IZDINXZHINXMIN-NEXT: fcvt.s.h a0, a0 -; CHECK32-IZDINXZHINXMIN-NEXT: lui a1, 391168 -; CHECK32-IZDINXZHINXMIN-NEXT: addi a1, a1, -1 -; CHECK32-IZDINXZHINXMIN-NEXT: fle.s a2, zero, a0 -; CHECK32-IZDINXZHINXMIN-NEXT: flt.s a1, a1, a0 -; CHECK32-IZDINXZHINXMIN-NEXT: neg s0, a1 -; CHECK32-IZDINXZHINXMIN-NEXT: neg s1, a2 +; CHECK32-IZDINXZHINXMIN-NEXT: fcvt.s.h s0, a0 +; CHECK32-IZDINXZHINXMIN-NEXT: fle.s a0, zero, s0 +; CHECK32-IZDINXZHINXMIN-NEXT: neg s1, a0 +; CHECK32-IZDINXZHINXMIN-NEXT: mv a0, s0 ; CHECK32-IZDINXZHINXMIN-NEXT: call __fixunssfdi ; CHECK32-IZDINXZHINXMIN-NEXT: and a0, s1, a0 +; CHECK32-IZDINXZHINXMIN-NEXT: lui a2, 391168 ; CHECK32-IZDINXZHINXMIN-NEXT: and a1, s1, a1 -; CHECK32-IZDINXZHINXMIN-NEXT: or a0, s0, a0 -; CHECK32-IZDINXZHINXMIN-NEXT: or a1, s0, a1 +; CHECK32-IZDINXZHINXMIN-NEXT: addi a2, a2, -1 +; CHECK32-IZDINXZHINXMIN-NEXT: flt.s a2, a2, s0 +; CHECK32-IZDINXZHINXMIN-NEXT: neg a2, a2 +; CHECK32-IZDINXZHINXMIN-NEXT: or a0, a2, a0 +; CHECK32-IZDINXZHINXMIN-NEXT: or a1, a2, a1 ; CHECK32-IZDINXZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; CHECK32-IZDINXZHINXMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; CHECK32-IZDINXZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload diff --git a/llvm/test/CodeGen/RISCV/rv64-half-convert.ll b/llvm/test/CodeGen/RISCV/rv64-half-convert.ll index 57061e1..f89d1abf 100644 --- a/llvm/test/CodeGen/RISCV/rv64-half-convert.ll +++ b/llvm/test/CodeGen/RISCV/rv64-half-convert.ll @@ -253,8 +253,8 @@ define i128 @fptosi_sat_f16_to_i128(half %a) nounwind { ; RV64IZHINX-NEXT: srli a1, a2, 1 ; RV64IZHINX-NEXT: .LBB4_4: ; RV64IZHINX-NEXT: feq.s a2, s0, s0 -; RV64IZHINX-NEXT: neg a3, a3 ; RV64IZHINX-NEXT: neg a4, s1 +; RV64IZHINX-NEXT: neg a3, a3 ; RV64IZHINX-NEXT: neg a2, a2 ; RV64IZHINX-NEXT: and a0, a4, a0 ; RV64IZHINX-NEXT: and a1, a2, a1 @@ -334,18 +334,19 @@ define i128 @fptoui_sat_f16_to_i128(half %a) nounwind { ; RV64IZHINX-NEXT: sd ra, 24(sp) # 8-byte Folded Spill ; RV64IZHINX-NEXT: sd s0, 16(sp) # 8-byte Folded Spill ; RV64IZHINX-NEXT: sd s1, 8(sp) # 8-byte Folded Spill -; RV64IZHINX-NEXT: fcvt.s.h a0, a0 -; RV64IZHINX-NEXT: lui a1, 522240 -; RV64IZHINX-NEXT: addi a1, a1, -1 -; RV64IZHINX-NEXT: fle.s a2, zero, a0 -; RV64IZHINX-NEXT: flt.s a1, a1, a0 -; RV64IZHINX-NEXT: neg s0, a1 -; RV64IZHINX-NEXT: neg s1, a2 +; RV64IZHINX-NEXT: fcvt.s.h s0, a0 +; RV64IZHINX-NEXT: fle.s a0, zero, s0 +; RV64IZHINX-NEXT: neg s1, a0 +; RV64IZHINX-NEXT: mv a0, s0 ; RV64IZHINX-NEXT: call __fixunssfti ; RV64IZHINX-NEXT: and a0, s1, a0 +; RV64IZHINX-NEXT: lui a2, 522240 ; RV64IZHINX-NEXT: and a1, s1, a1 -; RV64IZHINX-NEXT: or a0, s0, a0 -; RV64IZHINX-NEXT: or a1, s0, a1 +; RV64IZHINX-NEXT: addi a2, a2, -1 +; RV64IZHINX-NEXT: flt.s a2, a2, s0 +; RV64IZHINX-NEXT: neg a2, a2 +; RV64IZHINX-NEXT: or a0, a2, a0 +; RV64IZHINX-NEXT: or a1, a2, a1 ; RV64IZHINX-NEXT: ld ra, 24(sp) # 8-byte Folded Reload ; RV64IZHINX-NEXT: ld s0, 16(sp) # 8-byte Folded Reload ; RV64IZHINX-NEXT: ld s1, 8(sp) # 8-byte Folded Reload diff --git a/llvm/test/MC/ELF/many-instructions.s b/llvm/test/MC/ELF/many-instructions.s index 843d35f..7c13c0d 100644 --- a/llvm/test/MC/ELF/many-instructions.s +++ b/llvm/test/MC/ELF/many-instructions.s @@ -1,4 +1,5 @@ -# REQUIRES: asserts +## Checks the size of an internal MC structure that is different on 32-bit. +# REQUIRES: asserts, llvm-64-bits # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o /dev/null -debug-only=mc-dump 2>&1 | grep -E -o '[0-9]+ Data Size:[0-9]+' | FileCheck %s ## Test that encodeInstruction may cause a new fragment to be created. diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr151664-cost-hoisted-vector-scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr151664-cost-hoisted-vector-scalable.ll index 8495dee..b4df63d 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/pr151664-cost-hoisted-vector-scalable.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr151664-cost-hoisted-vector-scalable.ll @@ -1,47 +1,28 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 5 -; REQUIRES: asserts -; RUN: opt -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve -S \ -; RUN: -debug-only=loop-vectorize %s 2>&1 | FileCheck %s +; RUN: opt -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve -S %s | FileCheck %s -; FIXME: Hoisted vector code should be costed with scalable cost. -; In this example, `<vscale x 4 x float> @llvm.minimumnum` has an invalid cost, -; and hence should not be produced by LoopVectorize. - -; CHECK: LV: Found an estimated cost of Invalid for VF vscale x 4 For instruction: %res = tail call float @llvm.minimumnum.f32(float %arg, float 0.000000e+00) define void @cost_hoisted_vector_code(ptr %p, float %arg) { ; CHECK-LABEL: define void @cost_hoisted_vector_code( ; CHECK-SAME: ptr [[P:%.*]], float [[ARG:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8 ; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8 -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 -1, [[TMP3]] -; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 -1, [[N_MOD_VF]] -; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[ARG]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = add i64 1, [[N_VEC]] -; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x float> @llvm.minimumnum.nxv4f32(<vscale x 4 x float> [[BROADCAST_SPLAT]], <vscale x 4 x float> zeroinitializer) +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[ARG]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP0:%.*]] = call <4 x float> @llvm.minimumnum.v4f32(<4 x float> [[BROADCAST_SPLAT]], <4 x float> zeroinitializer) ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[INDEX:%.*]] = add i64 1, [[INDEX1]] ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[P]], i64 [[INDEX]] -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4 -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr float, ptr [[TMP8]], i64 [[TMP10]] -; CHECK-NEXT: store <vscale x 4 x float> [[TMP7]], ptr [[TMP8]], align 4 -; CHECK-NEXT: store <vscale x 4 x float> [[TMP7]], ptr [[TMP11]], align 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], [[TMP5]] -; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[TMP8]], i32 4 +; CHECK-NEXT: store <4 x float> [[TMP0]], ptr [[TMP8]], align 4 +; CHECK-NEXT: store <4 x float> [[TMP0]], ptr [[TMP2]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 8 +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], -8 +; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 -1, [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK-NEXT: br label %[[SCALAR_PH]] ; CHECK: [[SCALAR_PH]]: ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fixed-width-inorder-core.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fixed-width-inorder-core.ll index 20bc0af..76a7536 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fixed-width-inorder-core.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fixed-width-inorder-core.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 ; RUN: opt < %s -mtriple=aarch64-none-elf -mcpu=cortex-a510 -mattr=+sve -passes=loop-vectorize -S | FileCheck %s --check-prefix=CHECK-CA510 ; RUN: opt < %s -mtriple=aarch64-none-elf -mcpu=cortex-a520 -mattr=+sve -passes=loop-vectorize -S | FileCheck %s --check-prefix=CHECK-CA520 +; RUN: opt < %s -mtriple=aarch64-none-elf -mcpu=cortex-a320 -mattr=+sve -passes=loop-vectorize -S | FileCheck %s --check-prefix=CHECK-CA320 define void @sve_add(ptr %dst, ptr %a, ptr %b, i64 %n) { ; CHECK-CA510-LABEL: define void @sve_add( @@ -131,6 +132,70 @@ define void @sve_add(ptr %dst, ptr %a, ptr %b, i64 %n) { ; CHECK-CA520: [[FOR_COND_CLEANUP]]: ; CHECK-CA520-NEXT: ret void ; +; CHECK-CA320-LABEL: define void @sve_add( +; CHECK-CA320-SAME: ptr [[DST:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-CA320-NEXT: [[ENTRY:.*:]] +; CHECK-CA320-NEXT: [[B3:%.*]] = ptrtoint ptr [[B]] to i64 +; CHECK-CA320-NEXT: [[A2:%.*]] = ptrtoint ptr [[A]] to i64 +; CHECK-CA320-NEXT: [[DST1:%.*]] = ptrtoint ptr [[DST]] to i64 +; CHECK-CA320-NEXT: [[CMP9_NOT:%.*]] = icmp eq i64 [[N]], 0 +; CHECK-CA320-NEXT: br i1 [[CMP9_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]] +; CHECK-CA320: [[FOR_BODY_PREHEADER]]: +; CHECK-CA320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 +; CHECK-CA320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; CHECK-CA320: [[VECTOR_MEMCHECK]]: +; CHECK-CA320-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[A2]] +; CHECK-CA320-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 32 +; CHECK-CA320-NEXT: [[TMP1:%.*]] = sub i64 [[DST1]], [[B3]] +; CHECK-CA320-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP1]], 32 +; CHECK-CA320-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]] +; CHECK-CA320-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; CHECK-CA320: [[VECTOR_PH]]: +; CHECK-CA320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8 +; CHECK-CA320-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-CA320-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-CA320: [[VECTOR_BODY]]: +; CHECK-CA320-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-CA320-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]] +; CHECK-CA320-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw float, ptr [[TMP2]], i32 4 +; CHECK-CA320-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 4 +; CHECK-CA320-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP3]], align 4 +; CHECK-CA320-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]] +; CHECK-CA320-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw float, ptr [[TMP4]], i32 4 +; CHECK-CA320-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP4]], align 4 +; CHECK-CA320-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x float>, ptr [[TMP5]], align 4 +; CHECK-CA320-NEXT: [[TMP6:%.*]] = fadd fast <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD]] +; CHECK-CA320-NEXT: [[TMP7:%.*]] = fadd fast <4 x float> [[WIDE_LOAD7]], [[WIDE_LOAD5]] +; CHECK-CA320-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[DST]], i64 [[INDEX]] +; CHECK-CA320-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw float, ptr [[TMP8]], i32 4 +; CHECK-CA320-NEXT: store <4 x float> [[TMP6]], ptr [[TMP8]], align 4 +; CHECK-CA320-NEXT: store <4 x float> [[TMP7]], ptr [[TMP9]], align 4 +; CHECK-CA320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 +; CHECK-CA320-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-CA320-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-CA320: [[MIDDLE_BLOCK]]: +; CHECK-CA320-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-CA320-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]] +; CHECK-CA320: [[SCALAR_PH]]: +; CHECK-CA320-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; CHECK-CA320-NEXT: br label %[[FOR_BODY:.*]] +; CHECK-CA320: [[FOR_BODY]]: +; CHECK-CA320-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] +; CHECK-CA320-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]] +; CHECK-CA320-NEXT: [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4 +; CHECK-CA320-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDVARS_IV]] +; CHECK-CA320-NEXT: [[TMP12:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 +; CHECK-CA320-NEXT: [[ADD:%.*]] = fadd fast float [[TMP12]], [[TMP11]] +; CHECK-CA320-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw float, ptr [[DST]], i64 [[INDVARS_IV]] +; CHECK-CA320-NEXT: store float [[ADD]], ptr [[ARRAYIDX4]], align 4 +; CHECK-CA320-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 +; CHECK-CA320-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] +; CHECK-CA320-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-CA320: [[FOR_COND_CLEANUP_LOOPEXIT]]: +; CHECK-CA320-NEXT: br label %[[FOR_COND_CLEANUP]] +; CHECK-CA320: [[FOR_COND_CLEANUP]]: +; CHECK-CA320-NEXT: ret void +; entry: %cmp9.not = icmp eq i64 %n, 0 br i1 %cmp9.not, label %for.cond.cleanup, label %for.body @@ -160,3 +225,8 @@ for.cond.cleanup: ; preds = %for.cond.cleanup.lo ; CHECK-CA520: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} ; CHECK-CA520: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]} ;. +; CHECK-CA320: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} +; CHECK-CA320: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} +; CHECK-CA320: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} +; CHECK-CA320: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]} +;. diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll index ce7b78e..2b01018 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll @@ -1,81 +1,100 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5 ; RUN: opt -passes=loop-vectorize -S < %s | FileCheck %s target triple = "aarch64-unknown-linux-gnu" define void @trip7_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 { -; CHECK-LABEL: @trip7_i64( -; CHECK: = call i64 @llvm.vscale.i64() -; CHECK-NEXT: = mul nuw i64 -; CHECK: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[VF:%.*]] = mul nuw i64 [[VSCALE]], 2 -; CHECK: vector.body: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] -; CHECK: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ {{%.*}}, %vector.ph ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %vector.body ] -; CHECK: {{%.*}} = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr {{%.*}}, i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison) -; CHECK: {{%.*}} = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr {{%.*}}, i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison) -; CHECK: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> {{%.*}}, ptr {{%.*}}, i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[VF]] +; CHECK-LABEL: define void @trip7_i64( +; CHECK-SAME: ptr noalias noundef captures(none) [[DST:%.*]], ptr noalias noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], 1 +; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 7, [[TMP2]] +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2 +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 7) +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison) +; CHECK-NEXT: [[TMP6:%.*]] = shl nsw <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 1) +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison) +; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 2 x i64> [[WIDE_MASKED_LOAD1]], [[TMP6]] +; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP8]], ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 7) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NOT:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true) -; CHECK-NEXT: [[COND:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NOT]], i32 0 -; CHECK-NEXT: br i1 [[COND]], label %middle.block, label %vector.body +; CHECK-NEXT: [[TMP9:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true) +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[TMP9]], i32 0 +; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br [[EXIT:label %.*]] +; CHECK: [[SCALAR_PH]]: ; entry: - br label %for.body + br label %loop -for.body: ; preds = %entry, %for.body - %i.06 = phi i64 [ 0, %entry ], [ %inc, %for.body ] - %arrayidx = getelementptr inbounds i64, ptr %src, i64 %i.06 - %0 = load i64, ptr %arrayidx, align 8 +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep.src = getelementptr inbounds i64, ptr %src, i64 %iv + %0 = load i64, ptr %gep.src, align 8 %mul = shl nsw i64 %0, 1 - %arrayidx1 = getelementptr inbounds i64, ptr %dst, i64 %i.06 - %1 = load i64, ptr %arrayidx1, align 8 + %gep.dst = getelementptr inbounds i64, ptr %dst, i64 %iv + %1 = load i64, ptr %gep.dst, align 8 %add = add nsw i64 %1, %mul - store i64 %add, ptr %arrayidx1, align 8 - %inc = add nuw nsw i64 %i.06, 1 - %exitcond.not = icmp eq i64 %inc, 7 - br i1 %exitcond.not, label %for.end, label %for.body + store i64 %add, ptr %gep.dst, align 8 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 7 + br i1 %ec, label %exit, label %loop -for.end: ; preds = %for.body +exit: ret void } define void @trip5_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 { -; CHECK-LABEL: @trip5_i8( -; CHECK-NEXT: entry: -; CHECK-NEXT: br label [[FOR_BODY:%.*]] -; CHECK: for.body: -; CHECK-NEXT: [[I_08:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] -; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[I_08]] -; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-LABEL: define void @trip5_i8( +; CHECK-SAME: ptr noalias noundef captures(none) [[DST:%.*]], ptr noalias noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]] +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[GEP_SRC]], align 1 ; CHECK-NEXT: [[MUL:%.*]] = shl i8 [[TMP0]], 1 -; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[I_08]] -; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1 +; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]] +; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[GEP_DST]], align 1 ; CHECK-NEXT: [[ADD:%.*]] = add i8 [[MUL]], [[TMP1]] -; CHECK-NEXT: store i8 [[ADD]], ptr [[ARRAYIDX1]], align 1 -; CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_08]], 1 -; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 5 -; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] -; CHECK: for.end: +; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_DST]], align 1 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 5 +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]] +; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; entry: - br label %for.body + br label %loop -for.body: ; preds = %entry, %for.body - %i.08 = phi i64 [ 0, %entry ], [ %inc, %for.body ] - %arrayidx = getelementptr inbounds i8, ptr %src, i64 %i.08 - %0 = load i8, ptr %arrayidx, align 1 +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep.src = getelementptr inbounds i8, ptr %src, i64 %iv + %0 = load i8, ptr %gep.src, align 1 %mul = shl i8 %0, 1 - %arrayidx1 = getelementptr inbounds i8, ptr %dst, i64 %i.08 - %1 = load i8, ptr %arrayidx1, align 1 + %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv + %1 = load i8, ptr %gep.dst, align 1 %add = add i8 %mul, %1 - store i8 %add, ptr %arrayidx1, align 1 - %inc = add nuw nsw i64 %i.08, 1 - %exitcond.not = icmp eq i64 %inc, 5 - br i1 %exitcond.not, label %for.end, label %for.body + store i8 %add, ptr %gep.dst, align 1 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 5 + br i1 %ec, label %exit, label %loop -for.end: ; preds = %for.body +exit: ret void } diff --git a/llvm/test/Transforms/LoopVectorize/intrinsic.ll b/llvm/test/Transforms/LoopVectorize/intrinsic.ll index 9c910d7..10d83a4 100644 --- a/llvm/test/Transforms/LoopVectorize/intrinsic.ll +++ b/llvm/test/Transforms/LoopVectorize/intrinsic.ll @@ -324,6 +324,56 @@ for.end: ; preds = %for.body, %entry declare double @llvm.exp2.f64(double) +define void @ldexp_f32i32(i32 %n, ptr %y, ptr %x, i32 %exp) { +; CHECK-LABEL: @ldexp_f32i32( +; CHECK: llvm.ldexp.v4f32.v4i32 +; CHECK: ret void +; +entry: + br label %for.body + +for.body: ; preds = %entry, %for.body + %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ] + %arrayidx = getelementptr inbounds float, ptr %y, i32 %iv + %0 = load float, ptr %arrayidx, align 4 + %call = tail call float @llvm.ldexp.f32.i32(float %0, i32 %exp) + %arrayidx2 = getelementptr inbounds float, ptr %x, i32 %iv + store float %call, ptr %arrayidx2, align 4 + %iv.next = add i32 %iv, 1 + %exitcond = icmp eq i32 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.ldexp.f32.i32(float, i32) + +define void @ldexp_f64i32(i32 %n, ptr %y, ptr %x, i32 %exp) { +; CHECK-LABEL: @ldexp_f64i32( +; CHECK: llvm.ldexp.v4f64.v4i32 +; CHECK: ret void +; +entry: + br label %for.body + +for.body: ; preds = %entry, %for.body + %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ] + %arrayidx = getelementptr inbounds double, ptr %y, i32 %iv + %0 = load double, ptr %arrayidx, align 8 + %call = tail call double @llvm.ldexp.f64.i32(double %0, i32 %exp) + %arrayidx2 = getelementptr inbounds double, ptr %x, i32 %iv + store double %call, ptr %arrayidx2, align 8 + %iv.next = add i32 %iv, 1 + %exitcond = icmp eq i32 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.ldexp.f64.i32(double, i32) + define void @log_f32(i32 %n, ptr %y, ptr %x) { ; CHECK-LABEL: @log_f32( ; CHECK: llvm.log.v4f32 @@ -976,6 +1026,157 @@ for.end: ; preds = %for.body, %entry declare double @llvm.roundeven.f64(double) + +define void @lround_i32f32(i32 %n, ptr %y, ptr %x) { +; CHECK-LABEL: @lround_i32f32( +; CHECK: llvm.lround.v4i32.v4f32 +; CHECK: ret void +; +entry: + br label %for.body + +for.body: ; preds = %entry, %for.body + %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ] + %arrayidx = getelementptr inbounds float, ptr %y, i32 %iv + %0 = load float, ptr %arrayidx, align 4 + %call = tail call i32 @llvm.lround.i32.f32(float %0) + %arrayidx2 = getelementptr inbounds i32, ptr %x, i32 %iv + store i32 %call, ptr %arrayidx2, align 4 + %iv.next = add i32 %iv, 1 + %exitcond = icmp eq i32 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare i32 @llvm.lround.i32.f32(float) + +define void @lround_i32f64(i32 %n, ptr %y, ptr %x) { +; CHECK-LABEL: @lround_i32f64( +; CHECK: llvm.lround.v4i32.v4f64 +; CHECK: ret void +; +entry: + br label %for.body + +for.body: ; preds = %entry, %for.body + %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ] + %arrayidx = getelementptr inbounds double, ptr %y, i32 %iv + %0 = load double, ptr %arrayidx, align 8 + %call = tail call i32 @llvm.lround.i32.f64(double %0) + %arrayidx2 = getelementptr inbounds i32, ptr %x, i32 %iv + store i32 %call, ptr %arrayidx2, align 8 + %iv.next = add i32 %iv, 1 + %exitcond = icmp eq i32 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare i32 @llvm.lround.i32.f64(double) + +define void @lround_i64f32(i32 %n, ptr %y, ptr %x) { +; CHECK-LABEL: @lround_i64f32( +; CHECK: llvm.lround.v4i64.v4f32 +; CHECK: ret void +; +entry: + br label %for.body + +for.body: ; preds = %entry, %for.body + %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ] + %arrayidx = getelementptr inbounds float, ptr %y, i32 %iv + %0 = load float, ptr %arrayidx, align 4 + %call = tail call i64 @llvm.lround.i64.f32(float %0) + %arrayidx2 = getelementptr inbounds i64, ptr %x, i32 %iv + store i64 %call, ptr %arrayidx2, align 4 + %iv.next = add i32 %iv, 1 + %exitcond = icmp eq i32 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare i64 @llvm.lround.i64.f32(float) + +define void @lround_i64f64(i32 %n, ptr %y, ptr %x) { +; CHECK-LABEL: @lround_i64f64( +; CHECK: llvm.lround.v4i64.v4f64 +; CHECK: ret void +; +entry: + br label %for.body + +for.body: ; preds = %entry, %for.body + %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ] + %arrayidx = getelementptr inbounds double, ptr %y, i32 %iv + %0 = load double, ptr %arrayidx, align 8 + %call = tail call i64 @llvm.lround.i64.f64(double %0) + %arrayidx2 = getelementptr inbounds i64, ptr %x, i32 %iv + store i64 %call, ptr %arrayidx2, align 8 + %iv.next = add i32 %iv, 1 + %exitcond = icmp eq i32 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare i64 @llvm.lround.i64.f64(double) + +define void @llround_i64f32(i32 %n, ptr %y, ptr %x) { +; CHECK-LABEL: @llround_i64f32( +; CHECK: llvm.llround.v4i64.v4f32 +; CHECK: ret void +; +entry: + br label %for.body + +for.body: ; preds = %entry, %for.body + %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ] + %arrayidx = getelementptr inbounds float, ptr %y, i32 %iv + %0 = load float, ptr %arrayidx, align 4 + %call = tail call i64 @llvm.llround.i64.f32(float %0) + %arrayidx2 = getelementptr inbounds i64, ptr %x, i32 %iv + store i64 %call, ptr %arrayidx2, align 4 + %iv.next = add i32 %iv, 1 + %exitcond = icmp eq i32 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare i64 @llvm.llround.i64.f32(float) + +define void @llround_i64f64(i32 %n, ptr %y, ptr %x) { +; CHECK-LABEL: @llround_i64f64( +; CHECK: llvm.llround.v4i64.v4f64 +; CHECK: ret void +; +entry: + br label %for.body + +for.body: ; preds = %entry, %for.body + %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ] + %arrayidx = getelementptr inbounds double, ptr %y, i32 %iv + %0 = load double, ptr %arrayidx, align 8 + %call = tail call i64 @llvm.llround.i64.f64(double %0) + %arrayidx2 = getelementptr inbounds i64, ptr %x, i32 %iv + store i64 %call, ptr %arrayidx2, align 8 + %iv.next = add i32 %iv, 1 + %exitcond = icmp eq i32 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare i64 @llvm.llround.i64.f64(double) + define void @fma_f32(i32 %n, ptr %y, ptr %x, ptr %z, ptr %w) { ; CHECK-LABEL: @fma_f32( ; CHECK: llvm.fma.v4f32 diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/exp.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/exp.ll new file mode 100644 index 0000000..301e5da --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/exp.ll @@ -0,0 +1,279 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64 < %s | FileCheck %s + +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" + +define void @ldexp_f32i32(ptr %x, ptr %y, i32 %exp) { +; CHECK-LABEL: @ldexp_f32i32( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[L0:%.*]] = load float, ptr [[X:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 1 +; CHECK-NEXT: [[L2:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2 +; CHECK-NEXT: [[L4:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3 +; CHECK-NEXT: [[L6:%.*]] = load float, ptr [[ARRAYIDX_3]], align 4 +; CHECK-NEXT: [[L1:%.*]] = tail call float @llvm.ldexp.f32.i32(float [[L0]], i32 [[EXP:%.*]]) +; CHECK-NEXT: [[L3:%.*]] = tail call float @llvm.ldexp.f32.i32(float [[L2]], i32 [[EXP]]) +; CHECK-NEXT: [[L5:%.*]] = tail call float @llvm.ldexp.f32.i32(float [[L4]], i32 [[EXP]]) +; CHECK-NEXT: [[L7:%.*]] = tail call float @llvm.ldexp.f32.i32(float [[L6]], i32 [[EXP]]) +; CHECK-NEXT: store float [[L1]], ptr [[Y:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 1 +; CHECK-NEXT: store float [[L3]], ptr [[ARRAYIDX2_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 2 +; CHECK-NEXT: store float [[L5]], ptr [[ARRAYIDX2_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 3 +; CHECK-NEXT: store float [[L7]], ptr [[ARRAYIDX2_3]], align 4 +; CHECK-NEXT: ret void +; +entry: + %l0 = load float, ptr %x, align 4 + %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 1 + %l2 = load float, ptr %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds float, ptr %x, i64 2 + %l4 = load float, ptr %arrayidx.2, align 4 + %arrayidx.3 = getelementptr inbounds float, ptr %x, i64 3 + %l6 = load float, ptr %arrayidx.3, align 4 + %l1 = tail call float @llvm.ldexp.f32.i32(float %l0, i32 %exp) + %l3 = tail call float @llvm.ldexp.f32.i32(float %l2, i32 %exp) + %l5 = tail call float @llvm.ldexp.f32.i32(float %l4, i32 %exp) + %l7 = tail call float @llvm.ldexp.f32.i32(float %l6, i32 %exp) + store float %l1, ptr %y, align 4 + %arrayidx2.1 = getelementptr inbounds float, ptr %y, i64 1 + store float %l3, ptr %arrayidx2.1, align 4 + %arrayidx2.2 = getelementptr inbounds float, ptr %y, i64 2 + store float %l5, ptr %arrayidx2.2, align 4 + %arrayidx2.3 = getelementptr inbounds float, ptr %y, i64 3 + store float %l7, ptr %arrayidx2.3, align 4 + ret void +} + +define void @ldexp_f64i32(ptr %x, ptr %y, i32 %exp) { +; CHECK-LABEL: @ldexp_f64i32( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[X:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1 +; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2 +; CHECK-NEXT: [[L4:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3 +; CHECK-NEXT: [[L6:%.*]] = load double, ptr [[ARRAYIDX_3]], align 4 +; CHECK-NEXT: [[L1:%.*]] = tail call double @llvm.ldexp.f64.i32(double [[L0]], i32 [[EXP:%.*]]) +; CHECK-NEXT: [[L3:%.*]] = tail call double @llvm.ldexp.f64.i32(double [[L2]], i32 [[EXP]]) +; CHECK-NEXT: [[L5:%.*]] = tail call double @llvm.ldexp.f64.i32(double [[L4]], i32 [[EXP]]) +; CHECK-NEXT: [[L7:%.*]] = tail call double @llvm.ldexp.f64.i32(double [[L6]], i32 [[EXP]]) +; CHECK-NEXT: store double [[L1]], ptr [[Y:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 1 +; CHECK-NEXT: store double [[L3]], ptr [[ARRAYIDX2_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 2 +; CHECK-NEXT: store double [[L5]], ptr [[ARRAYIDX2_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 3 +; CHECK-NEXT: store double [[L7]], ptr [[ARRAYIDX2_3]], align 4 +; CHECK-NEXT: ret void +; +entry: + %l0 = load double, ptr %x, align 4 + %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 1 + %l2 = load double, ptr %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds double, ptr %x, i64 2 + %l4 = load double, ptr %arrayidx.2, align 4 + %arrayidx.3 = getelementptr inbounds double, ptr %x, i64 3 + %l6 = load double, ptr %arrayidx.3, align 4 + %l1 = tail call double @llvm.ldexp.f64.i32(double %l0, i32 %exp) + %l3 = tail call double @llvm.ldexp.f64.i32(double %l2, i32 %exp) + %l5 = tail call double @llvm.ldexp.f64.i32(double %l4, i32 %exp) + %l7 = tail call double @llvm.ldexp.f64.i32(double %l6, i32 %exp) + store double %l1, ptr %y, align 4 + %arrayidx2.1 = getelementptr inbounds double, ptr %y, i64 1 + store double %l3, ptr %arrayidx2.1, align 4 + %arrayidx2.2 = getelementptr inbounds double, ptr %y, i64 2 + store double %l5, ptr %arrayidx2.2, align 4 + %arrayidx2.3 = getelementptr inbounds double, ptr %y, i64 3 + store double %l7, ptr %arrayidx2.3, align 4 + ret void +} + +define void @ldexp_f32i64(ptr %x, ptr %y, i64 %exp) { +; CHECK-LABEL: @ldexp_f32i64( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[L0:%.*]] = load float, ptr [[X:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 1 +; CHECK-NEXT: [[L2:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2 +; CHECK-NEXT: [[L4:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3 +; CHECK-NEXT: [[L6:%.*]] = load float, ptr [[ARRAYIDX_3]], align 4 +; CHECK-NEXT: [[L1:%.*]] = tail call float @llvm.ldexp.f32.i64(float [[L0]], i64 [[EXP:%.*]]) +; CHECK-NEXT: [[L3:%.*]] = tail call float @llvm.ldexp.f32.i64(float [[L2]], i64 [[EXP]]) +; CHECK-NEXT: [[L5:%.*]] = tail call float @llvm.ldexp.f32.i64(float [[L4]], i64 [[EXP]]) +; CHECK-NEXT: [[L7:%.*]] = tail call float @llvm.ldexp.f32.i64(float [[L6]], i64 [[EXP]]) +; CHECK-NEXT: store float [[L1]], ptr [[Y:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 1 +; CHECK-NEXT: store float [[L3]], ptr [[ARRAYIDX2_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 2 +; CHECK-NEXT: store float [[L5]], ptr [[ARRAYIDX2_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 3 +; CHECK-NEXT: store float [[L7]], ptr [[ARRAYIDX2_3]], align 4 +; CHECK-NEXT: ret void +; +entry: + %l0 = load float, ptr %x, align 4 + %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 1 + %l2 = load float, ptr %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds float, ptr %x, i64 2 + %l4 = load float, ptr %arrayidx.2, align 4 + %arrayidx.3 = getelementptr inbounds float, ptr %x, i64 3 + %l6 = load float, ptr %arrayidx.3, align 4 + %l1 = tail call float @llvm.ldexp.f32.i64(float %l0, i64 %exp) + %l3 = tail call float @llvm.ldexp.f32.i64(float %l2, i64 %exp) + %l5 = tail call float @llvm.ldexp.f32.i64(float %l4, i64 %exp) + %l7 = tail call float @llvm.ldexp.f32.i64(float %l6, i64 %exp) + store float %l1, ptr %y, align 4 + %arrayidx2.1 = getelementptr inbounds float, ptr %y, i64 1 + store float %l3, ptr %arrayidx2.1, align 4 + %arrayidx2.2 = getelementptr inbounds float, ptr %y, i64 2 + store float %l5, ptr %arrayidx2.2, align 4 + %arrayidx2.3 = getelementptr inbounds float, ptr %y, i64 3 + store float %l7, ptr %arrayidx2.3, align 4 + ret void +} + +define void @ldexp_f64i64(ptr %x, ptr %y, i64 %exp) { +; CHECK-LABEL: @ldexp_f64i64( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[X:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1 +; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2 +; CHECK-NEXT: [[L4:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3 +; CHECK-NEXT: [[L6:%.*]] = load double, ptr [[ARRAYIDX_3]], align 4 +; CHECK-NEXT: [[L1:%.*]] = tail call double @llvm.ldexp.f64.i64(double [[L0]], i64 [[EXP:%.*]]) +; CHECK-NEXT: [[L3:%.*]] = tail call double @llvm.ldexp.f64.i64(double [[L2]], i64 [[EXP]]) +; CHECK-NEXT: [[L5:%.*]] = tail call double @llvm.ldexp.f64.i64(double [[L4]], i64 [[EXP]]) +; CHECK-NEXT: [[L7:%.*]] = tail call double @llvm.ldexp.f64.i64(double [[L6]], i64 [[EXP]]) +; CHECK-NEXT: store double [[L1]], ptr [[Y:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 1 +; CHECK-NEXT: store double [[L3]], ptr [[ARRAYIDX2_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 2 +; CHECK-NEXT: store double [[L5]], ptr [[ARRAYIDX2_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 3 +; CHECK-NEXT: store double [[L7]], ptr [[ARRAYIDX2_3]], align 4 +; CHECK-NEXT: ret void +; +entry: + %l0 = load double, ptr %x, align 4 + %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 1 + %l2 = load double, ptr %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds double, ptr %x, i64 2 + %l4 = load double, ptr %arrayidx.2, align 4 + %arrayidx.3 = getelementptr inbounds double, ptr %x, i64 3 + %l6 = load double, ptr %arrayidx.3, align 4 + %l1 = tail call double @llvm.ldexp.f64.i64(double %l0, i64 %exp) + %l3 = tail call double @llvm.ldexp.f64.i64(double %l2, i64 %exp) + %l5 = tail call double @llvm.ldexp.f64.i64(double %l4, i64 %exp) + %l7 = tail call double @llvm.ldexp.f64.i64(double %l6, i64 %exp) + store double %l1, ptr %y, align 4 + %arrayidx2.1 = getelementptr inbounds double, ptr %y, i64 1 + store double %l3, ptr %arrayidx2.1, align 4 + %arrayidx2.2 = getelementptr inbounds double, ptr %y, i64 2 + store double %l5, ptr %arrayidx2.2, align 4 + %arrayidx2.3 = getelementptr inbounds double, ptr %y, i64 3 + store double %l7, ptr %arrayidx2.3, align 4 + ret void +} + +define void @ldexp_f32i32_i64(ptr %x, ptr %y, i32 %exp32, i64 %exp64) { +; CHECK-LABEL: @ldexp_f32i32_i64( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[L0:%.*]] = load float, ptr [[X:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 1 +; CHECK-NEXT: [[L2:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2 +; CHECK-NEXT: [[L4:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3 +; CHECK-NEXT: [[L6:%.*]] = load float, ptr [[ARRAYIDX_3]], align 4 +; CHECK-NEXT: [[L1:%.*]] = tail call float @llvm.ldexp.f32.i32(float [[L0]], i32 [[EXP32:%.*]]) +; CHECK-NEXT: [[L3:%.*]] = tail call float @llvm.ldexp.f32.i32(float [[L2]], i32 [[EXP32]]) +; CHECK-NEXT: [[L5:%.*]] = tail call float @llvm.ldexp.f32.i64(float [[L4]], i64 [[EXP64:%.*]]) +; CHECK-NEXT: [[L7:%.*]] = tail call float @llvm.ldexp.f32.i64(float [[L6]], i64 [[EXP64]]) +; CHECK-NEXT: store float [[L1]], ptr [[Y:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 1 +; CHECK-NEXT: store float [[L3]], ptr [[ARRAYIDX2_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 2 +; CHECK-NEXT: store float [[L5]], ptr [[ARRAYIDX2_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 3 +; CHECK-NEXT: store float [[L7]], ptr [[ARRAYIDX2_3]], align 4 +; CHECK-NEXT: ret void +; +entry: + %l0 = load float, ptr %x, align 4 + %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 1 + %l2 = load float, ptr %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds float, ptr %x, i64 2 + %l4 = load float, ptr %arrayidx.2, align 4 + %arrayidx.3 = getelementptr inbounds float, ptr %x, i64 3 + %l6 = load float, ptr %arrayidx.3, align 4 + %l1 = tail call float @llvm.ldexp.f32.i32(float %l0, i32 %exp32) + %l3 = tail call float @llvm.ldexp.f32.i32(float %l2, i32 %exp32) + %l5 = tail call float @llvm.ldexp.f32.i64(float %l4, i64 %exp64) + %l7 = tail call float @llvm.ldexp.f32.i64(float %l6, i64 %exp64) + store float %l1, ptr %y, align 4 + %arrayidx2.1 = getelementptr inbounds float, ptr %y, i64 1 + store float %l3, ptr %arrayidx2.1, align 4 + %arrayidx2.2 = getelementptr inbounds float, ptr %y, i64 2 + store float %l5, ptr %arrayidx2.2, align 4 + %arrayidx2.3 = getelementptr inbounds float, ptr %y, i64 3 + store float %l7, ptr %arrayidx2.3, align 4 + ret void +} + +define void @ldexp_f64_i32_i64(ptr %x, ptr %y, i32 %exp32, i64 %exp64) { +; CHECK-LABEL: @ldexp_f64_i32_i64( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[X:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1 +; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2 +; CHECK-NEXT: [[L4:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3 +; CHECK-NEXT: [[L6:%.*]] = load double, ptr [[ARRAYIDX_3]], align 4 +; CHECK-NEXT: [[L1:%.*]] = tail call double @llvm.ldexp.f64.i32(double [[L0]], i32 [[EXP32:%.*]]) +; CHECK-NEXT: [[L3:%.*]] = tail call double @llvm.ldexp.f64.i32(double [[L2]], i32 [[EXP32]]) +; CHECK-NEXT: [[L5:%.*]] = tail call double @llvm.ldexp.f64.i64(double [[L4]], i64 [[EXP64:%.*]]) +; CHECK-NEXT: [[L7:%.*]] = tail call double @llvm.ldexp.f64.i64(double [[L6]], i64 [[EXP64]]) +; CHECK-NEXT: store double [[L1]], ptr [[Y:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 1 +; CHECK-NEXT: store double [[L3]], ptr [[ARRAYIDX2_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 2 +; CHECK-NEXT: store double [[L5]], ptr [[ARRAYIDX2_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 3 +; CHECK-NEXT: store double [[L7]], ptr [[ARRAYIDX2_3]], align 4 +; CHECK-NEXT: ret void +; +entry: + %l0 = load double, ptr %x, align 4 + %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 1 + %l2 = load double, ptr %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds double, ptr %x, i64 2 + %l4 = load double, ptr %arrayidx.2, align 4 + %arrayidx.3 = getelementptr inbounds double, ptr %x, i64 3 + %l6 = load double, ptr %arrayidx.3, align 4 + %l1 = tail call double @llvm.ldexp.f64.i32(double %l0, i32 %exp32) + %l3 = tail call double @llvm.ldexp.f64.i32(double %l2, i32 %exp32) + %l5 = tail call double @llvm.ldexp.f64.i64(double %l4, i64 %exp64) + %l7 = tail call double @llvm.ldexp.f64.i64(double %l6, i64 %exp64) + store double %l1, ptr %y, align 4 + %arrayidx2.1 = getelementptr inbounds double, ptr %y, i64 1 + store double %l3, ptr %arrayidx2.1, align 4 + %arrayidx2.2 = getelementptr inbounds double, ptr %y, i64 2 + store double %l5, ptr %arrayidx2.2, align 4 + %arrayidx2.3 = getelementptr inbounds double, ptr %y, i64 3 + store double %l7, ptr %arrayidx2.3, align 4 + ret void +} + +declare float @llvm.ldexp.f32.i32(float, i32) +declare double @llvm.ldexp.f64.i32(double, i32) +declare float @llvm.ldexp.f32.i64(float, i64) +declare double @llvm.ldexp.f64.i64(double, i64) diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fround.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fround.ll new file mode 100644 index 0000000..07a3fe7 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fround.ll @@ -0,0 +1,280 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64 < %s | FileCheck %s + +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" + +define void @lround_i32f32(ptr %x, ptr %y, i32 %n) { +; CHECK-LABEL: @lround_i32f32( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[L0:%.*]] = load float, ptr [[X:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 1 +; CHECK-NEXT: [[L2:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2 +; CHECK-NEXT: [[L4:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3 +; CHECK-NEXT: [[L6:%.*]] = load float, ptr [[ARRAYIDX_3]], align 4 +; CHECK-NEXT: [[L1:%.*]] = tail call i32 @llvm.lround.i32.f32(float [[L0]]) +; CHECK-NEXT: [[L3:%.*]] = tail call i32 @llvm.lround.i32.f32(float [[L2]]) +; CHECK-NEXT: [[L5:%.*]] = tail call i32 @llvm.lround.i32.f32(float [[L4]]) +; CHECK-NEXT: [[L7:%.*]] = tail call i32 @llvm.lround.i32.f32(float [[L6]]) +; CHECK-NEXT: store i32 [[L1]], ptr [[Y:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 1 +; CHECK-NEXT: store i32 [[L3]], ptr [[ARRAYIDX2_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 2 +; CHECK-NEXT: store i32 [[L5]], ptr [[ARRAYIDX2_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 3 +; CHECK-NEXT: store i32 [[L7]], ptr [[ARRAYIDX2_3]], align 4 +; CHECK-NEXT: ret void +; +entry: + %l0 = load float, ptr %x, align 4 + %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 1 + %l2 = load float, ptr %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds float, ptr %x, i64 2 + %l4 = load float, ptr %arrayidx.2, align 4 + %arrayidx.3 = getelementptr inbounds float, ptr %x, i64 3 + %l6 = load float, ptr %arrayidx.3, align 4 + %l1 = tail call i32 @llvm.lround.i32.f32(float %l0) + %l3 = tail call i32 @llvm.lround.i32.f32(float %l2) + %l5 = tail call i32 @llvm.lround.i32.f32(float %l4) + %l7 = tail call i32 @llvm.lround.i32.f32(float %l6) + store i32 %l1, ptr %y, align 4 + %arrayidx2.1 = getelementptr inbounds i32, ptr %y, i64 1 + store i32 %l3, ptr %arrayidx2.1, align 4 + %arrayidx2.2 = getelementptr inbounds i32, ptr %y, i64 2 + store i32 %l5, ptr %arrayidx2.2, align 4 + %arrayidx2.3 = getelementptr inbounds i32, ptr %y, i64 3 + store i32 %l7, ptr %arrayidx2.3, align 4 + ret void +} + +define void @lround_i32f64(ptr %x, ptr %y, i32 %n) { +; CHECK-LABEL: @lround_i32f64( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[X:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1 +; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2 +; CHECK-NEXT: [[L4:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3 +; CHECK-NEXT: [[L6:%.*]] = load double, ptr [[ARRAYIDX_3]], align 4 +; CHECK-NEXT: [[L1:%.*]] = tail call i32 @llvm.lround.i32.f64(double [[L0]]) +; CHECK-NEXT: [[L3:%.*]] = tail call i32 @llvm.lround.i32.f64(double [[L2]]) +; CHECK-NEXT: [[L5:%.*]] = tail call i32 @llvm.lround.i32.f64(double [[L4]]) +; CHECK-NEXT: [[L7:%.*]] = tail call i32 @llvm.lround.i32.f64(double [[L6]]) +; CHECK-NEXT: store i32 [[L1]], ptr [[Y:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 1 +; CHECK-NEXT: store i32 [[L3]], ptr [[ARRAYIDX2_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 2 +; CHECK-NEXT: store i32 [[L5]], ptr [[ARRAYIDX2_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 3 +; CHECK-NEXT: store i32 [[L7]], ptr [[ARRAYIDX2_3]], align 4 +; CHECK-NEXT: ret void +; +entry: + %l0 = load double, ptr %x, align 4 + %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 1 + %l2 = load double, ptr %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds double, ptr %x, i64 2 + %l4 = load double, ptr %arrayidx.2, align 4 + %arrayidx.3 = getelementptr inbounds double, ptr %x, i64 3 + %l6 = load double, ptr %arrayidx.3, align 4 + %l1 = tail call i32 @llvm.lround.i32.f64(double %l0) + %l3 = tail call i32 @llvm.lround.i32.f64(double %l2) + %l5 = tail call i32 @llvm.lround.i32.f64(double %l4) + %l7 = tail call i32 @llvm.lround.i32.f64(double %l6) + store i32 %l1, ptr %y, align 4 + %arrayidx2.1 = getelementptr inbounds i32, ptr %y, i64 1 + store i32 %l3, ptr %arrayidx2.1, align 4 + %arrayidx2.2 = getelementptr inbounds i32, ptr %y, i64 2 + store i32 %l5, ptr %arrayidx2.2, align 4 + %arrayidx2.3 = getelementptr inbounds i32, ptr %y, i64 3 + store i32 %l7, ptr %arrayidx2.3, align 4 + ret void +} + +define void @lround_i64f32(ptr %x, ptr %y, i64 %n) { +; CHECK-LABEL: @lround_i64f32( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[L0:%.*]] = load float, ptr [[X:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 1 +; CHECK-NEXT: [[L2:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2 +; CHECK-NEXT: [[L4:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3 +; CHECK-NEXT: [[L6:%.*]] = load float, ptr [[ARRAYIDX_3]], align 4 +; CHECK-NEXT: [[L1:%.*]] = tail call i64 @llvm.lround.i64.f32(float [[L0]]) +; CHECK-NEXT: [[L3:%.*]] = tail call i64 @llvm.lround.i64.f32(float [[L2]]) +; CHECK-NEXT: [[L5:%.*]] = tail call i64 @llvm.lround.i64.f32(float [[L4]]) +; CHECK-NEXT: [[L7:%.*]] = tail call i64 @llvm.lround.i64.f32(float [[L6]]) +; CHECK-NEXT: store i64 [[L1]], ptr [[Y:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 1 +; CHECK-NEXT: store i64 [[L3]], ptr [[ARRAYIDX2_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 2 +; CHECK-NEXT: store i64 [[L5]], ptr [[ARRAYIDX2_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 3 +; CHECK-NEXT: store i64 [[L7]], ptr [[ARRAYIDX2_3]], align 4 +; CHECK-NEXT: ret void +; +entry: + %l0 = load float, ptr %x, align 4 + %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 1 + %l2 = load float, ptr %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds float, ptr %x, i64 2 + %l4 = load float, ptr %arrayidx.2, align 4 + %arrayidx.3 = getelementptr inbounds float, ptr %x, i64 3 + %l6 = load float, ptr %arrayidx.3, align 4 + %l1 = tail call i64 @llvm.lround.i64.f32(float %l0) + %l3 = tail call i64 @llvm.lround.i64.f32(float %l2) + %l5 = tail call i64 @llvm.lround.i64.f32(float %l4) + %l7 = tail call i64 @llvm.lround.i64.f32(float %l6) + store i64 %l1, ptr %y, align 4 + %arrayidx2.1 = getelementptr inbounds i64, ptr %y, i64 1 + store i64 %l3, ptr %arrayidx2.1, align 4 + %arrayidx2.2 = getelementptr inbounds i64, ptr %y, i64 2 + store i64 %l5, ptr %arrayidx2.2, align 4 + %arrayidx2.3 = getelementptr inbounds i64, ptr %y, i64 3 + store i64 %l7, ptr %arrayidx2.3, align 4 + ret void +} + +define void @lround_i64f64(ptr %x, ptr %y, i64 %n) { +; CHECK-LABEL: @lround_i64f64( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[X:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1 +; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2 +; CHECK-NEXT: [[L4:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3 +; CHECK-NEXT: [[L6:%.*]] = load double, ptr [[ARRAYIDX_3]], align 4 +; CHECK-NEXT: [[L1:%.*]] = tail call i64 @llvm.lround.i64.f64(double [[L0]]) +; CHECK-NEXT: [[L3:%.*]] = tail call i64 @llvm.lround.i64.f64(double [[L2]]) +; CHECK-NEXT: [[L5:%.*]] = tail call i64 @llvm.lround.i64.f64(double [[L4]]) +; CHECK-NEXT: [[L7:%.*]] = tail call i64 @llvm.lround.i64.f64(double [[L6]]) +; CHECK-NEXT: store i64 [[L1]], ptr [[Y:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 1 +; CHECK-NEXT: store i64 [[L3]], ptr [[ARRAYIDX2_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 2 +; CHECK-NEXT: store i64 [[L5]], ptr [[ARRAYIDX2_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 3 +; CHECK-NEXT: store i64 [[L7]], ptr [[ARRAYIDX2_3]], align 4 +; CHECK-NEXT: ret void +; +entry: + %l0 = load double, ptr %x, align 4 + %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 1 + %l2 = load double, ptr %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds double, ptr %x, i64 2 + %l4 = load double, ptr %arrayidx.2, align 4 + %arrayidx.3 = getelementptr inbounds double, ptr %x, i64 3 + %l6 = load double, ptr %arrayidx.3, align 4 + %l1 = tail call i64 @llvm.lround.i64.f64(double %l0) + %l3 = tail call i64 @llvm.lround.i64.f64(double %l2) + %l5 = tail call i64 @llvm.lround.i64.f64(double %l4) + %l7 = tail call i64 @llvm.lround.i64.f64(double %l6) + store i64 %l1, ptr %y, align 4 + %arrayidx2.1 = getelementptr inbounds i64, ptr %y, i64 1 + store i64 %l3, ptr %arrayidx2.1, align 4 + %arrayidx2.2 = getelementptr inbounds i64, ptr %y, i64 2 + store i64 %l5, ptr %arrayidx2.2, align 4 + %arrayidx2.3 = getelementptr inbounds i64, ptr %y, i64 3 + store i64 %l7, ptr %arrayidx2.3, align 4 + ret void +} + +define void @llround_i64f32(ptr %x, ptr %y, i64 %n) { +; CHECK-LABEL: @llround_i64f32( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[L0:%.*]] = load float, ptr [[X:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 1 +; CHECK-NEXT: [[L2:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2 +; CHECK-NEXT: [[L4:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3 +; CHECK-NEXT: [[L6:%.*]] = load float, ptr [[ARRAYIDX_3]], align 4 +; CHECK-NEXT: [[L1:%.*]] = tail call i64 @llvm.llround.i64.f32(float [[L0]]) +; CHECK-NEXT: [[L3:%.*]] = tail call i64 @llvm.llround.i64.f32(float [[L2]]) +; CHECK-NEXT: [[L5:%.*]] = tail call i64 @llvm.llround.i64.f32(float [[L4]]) +; CHECK-NEXT: [[L7:%.*]] = tail call i64 @llvm.llround.i64.f32(float [[L6]]) +; CHECK-NEXT: store i64 [[L1]], ptr [[Y:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 1 +; CHECK-NEXT: store i64 [[L3]], ptr [[ARRAYIDX2_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 2 +; CHECK-NEXT: store i64 [[L5]], ptr [[ARRAYIDX2_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 3 +; CHECK-NEXT: store i64 [[L7]], ptr [[ARRAYIDX2_3]], align 4 +; CHECK-NEXT: ret void +; +entry: + %l0 = load float, ptr %x, align 4 + %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 1 + %l2 = load float, ptr %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds float, ptr %x, i64 2 + %l4 = load float, ptr %arrayidx.2, align 4 + %arrayidx.3 = getelementptr inbounds float, ptr %x, i64 3 + %l6 = load float, ptr %arrayidx.3, align 4 + %l1 = tail call i64 @llvm.llround.i64.f32(float %l0) + %l3 = tail call i64 @llvm.llround.i64.f32(float %l2) + %l5 = tail call i64 @llvm.llround.i64.f32(float %l4) + %l7 = tail call i64 @llvm.llround.i64.f32(float %l6) + store i64 %l1, ptr %y, align 4 + %arrayidx2.1 = getelementptr inbounds i64, ptr %y, i64 1 + store i64 %l3, ptr %arrayidx2.1, align 4 + %arrayidx2.2 = getelementptr inbounds i64, ptr %y, i64 2 + store i64 %l5, ptr %arrayidx2.2, align 4 + %arrayidx2.3 = getelementptr inbounds i64, ptr %y, i64 3 + store i64 %l7, ptr %arrayidx2.3, align 4 + ret void +} + +define void @llround_i64f64(ptr %x, ptr %y, i64 %n) { +; CHECK-LABEL: @llround_i64f64( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[X:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1 +; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2 +; CHECK-NEXT: [[L4:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3 +; CHECK-NEXT: [[L6:%.*]] = load double, ptr [[ARRAYIDX_3]], align 4 +; CHECK-NEXT: [[L1:%.*]] = tail call i64 @llvm.llround.i64.f64(double [[L0]]) +; CHECK-NEXT: [[L3:%.*]] = tail call i64 @llvm.llround.i64.f64(double [[L2]]) +; CHECK-NEXT: [[L5:%.*]] = tail call i64 @llvm.llround.i64.f64(double [[L4]]) +; CHECK-NEXT: [[L7:%.*]] = tail call i64 @llvm.llround.i64.f64(double [[L6]]) +; CHECK-NEXT: store i64 [[L1]], ptr [[Y:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 1 +; CHECK-NEXT: store i64 [[L3]], ptr [[ARRAYIDX2_1]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 2 +; CHECK-NEXT: store i64 [[L5]], ptr [[ARRAYIDX2_2]], align 4 +; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 3 +; CHECK-NEXT: store i64 [[L7]], ptr [[ARRAYIDX2_3]], align 4 +; CHECK-NEXT: ret void +; +entry: + %l0 = load double, ptr %x, align 4 + %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 1 + %l2 = load double, ptr %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds double, ptr %x, i64 2 + %l4 = load double, ptr %arrayidx.2, align 4 + %arrayidx.3 = getelementptr inbounds double, ptr %x, i64 3 + %l6 = load double, ptr %arrayidx.3, align 4 + %l1 = tail call i64 @llvm.llround.i64.f64(double %l0) + %l3 = tail call i64 @llvm.llround.i64.f64(double %l2) + %l5 = tail call i64 @llvm.llround.i64.f64(double %l4) + %l7 = tail call i64 @llvm.llround.i64.f64(double %l6) + store i64 %l1, ptr %y, align 4 + %arrayidx2.1 = getelementptr inbounds i64, ptr %y, i64 1 + store i64 %l3, ptr %arrayidx2.1, align 4 + %arrayidx2.2 = getelementptr inbounds i64, ptr %y, i64 2 + store i64 %l5, ptr %arrayidx2.2, align 4 + %arrayidx2.3 = getelementptr inbounds i64, ptr %y, i64 3 + store i64 %l7, ptr %arrayidx2.3, align 4 + ret void +} + +declare i32 @llvm.lround.i32.f32(float) +declare i64 @llvm.lround.i64.f32(float) +declare i64 @llvm.lround.i64.f64(double) +declare i64 @llvm.llround.i64.f32(float) +declare i64 @llvm.llround.i64.f64(double) diff --git a/llvm/test/Transforms/Scalarizer/intrinsics.ll b/llvm/test/Transforms/Scalarizer/intrinsics.ll index cee44ef..070c765 100644 --- a/llvm/test/Transforms/Scalarizer/intrinsics.ll +++ b/llvm/test/Transforms/Scalarizer/intrinsics.ll @@ -8,6 +8,7 @@ declare <2 x float> @llvm.sqrt.v2f32(<2 x float>) declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>) declare <2 x float> @llvm.minimum.v2f32(<2 x float>, <2 x float>) declare <2 x float> @llvm.maximum.v2f32(<2 x float>, <2 x float>) +declare <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float>, <2 x i32>) ; Ternary fp declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>) @@ -32,6 +33,8 @@ declare <2 x i32> @llvm.fptoui.sat.v2i32.v2f32(<2 x float>) ; Unary fp operand, int return type declare <2 x i32> @llvm.lrint.v2i32.v2f32(<2 x float>) declare <2 x i32> @llvm.llrint.v2i32.v2f32(<2 x float>) +declare <2 x i32> @llvm.lround.v2i32.v2f32(<2 x float>) +declare <2 x i32> @llvm.llround.v2i32.v2f32(<2 x float>) ; Bool return type, overloaded on fp operand type declare <2 x i1> @llvm.is.fpclass(<2 x float>, i32) @@ -159,6 +162,22 @@ define <2 x float> @scalarize_powi_v2f32(<2 x float> %x, i32 %y) #0 { ret <2 x float> %powi } +define <2 x float> @scalarize_ldexp_v2f32(<2 x float> %x, <2 x i32> %y) #0 { +; CHECK-LABEL: @scalarize_ldexp_v2f32( +; CHECK-NEXT: [[X_I0:%.*]] = extractelement <2 x float> [[X:%.*]], i64 0 +; CHECK-NEXT: [[Y:%.*]] = extractelement <2 x i32> [[Y1:%.*]], i64 0 +; CHECK-NEXT: [[POWI_I0:%.*]] = call float @llvm.ldexp.f32.i32(float [[X_I0]], i32 [[Y]]) +; CHECK-NEXT: [[X_I1:%.*]] = extractelement <2 x float> [[X]], i64 1 +; CHECK-NEXT: [[Y_I1:%.*]] = extractelement <2 x i32> [[Y1]], i64 1 +; CHECK-NEXT: [[POWI_I1:%.*]] = call float @llvm.ldexp.f32.i32(float [[X_I1]], i32 [[Y_I1]]) +; CHECK-NEXT: [[POWI_UPTO0:%.*]] = insertelement <2 x float> poison, float [[POWI_I0]], i64 0 +; CHECK-NEXT: [[POWI:%.*]] = insertelement <2 x float> [[POWI_UPTO0]], float [[POWI_I1]], i64 1 +; CHECK-NEXT: ret <2 x float> [[POWI]] +; + %powi = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> %x, <2 x i32> %y) + ret <2 x float> %powi +} + define <2 x i32> @scalarize_smul_fix_sat_v2i32(<2 x i32> %x) #0 { ; CHECK-LABEL: @scalarize_smul_fix_sat_v2i32( ; CHECK-NEXT: [[X_I0:%.*]] = extractelement <2 x i32> [[X:%.*]], i64 0 @@ -243,6 +262,34 @@ define <2 x i32> @scalarize_llrint(<2 x float> %x) #0 { ret <2 x i32> %rnd } +define <2 x i32> @scalarize_lround(<2 x float> %x) #0 { +; CHECK-LABEL: @scalarize_lround( +; CHECK-NEXT: [[X_I0:%.*]] = extractelement <2 x float> [[X:%.*]], i64 0 +; CHECK-NEXT: [[RND_I0:%.*]] = call i32 @llvm.lround.i32.f32(float [[X_I0]]) +; CHECK-NEXT: [[X_I1:%.*]] = extractelement <2 x float> [[X]], i64 1 +; CHECK-NEXT: [[RND_I1:%.*]] = call i32 @llvm.lround.i32.f32(float [[X_I1]]) +; CHECK-NEXT: [[RND_UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[RND_I0]], i64 0 +; CHECK-NEXT: [[RND:%.*]] = insertelement <2 x i32> [[RND_UPTO0]], i32 [[RND_I1]], i64 1 +; CHECK-NEXT: ret <2 x i32> [[RND]] +; + %rnd = call <2 x i32> @llvm.lround.v2i32.v2f32(<2 x float> %x) + ret <2 x i32> %rnd +} + +define <2 x i32> @scalarize_llround(<2 x float> %x) #0 { +; CHECK-LABEL: @scalarize_llround( +; CHECK-NEXT: [[X_I0:%.*]] = extractelement <2 x float> [[X:%.*]], i64 0 +; CHECK-NEXT: [[RND_I0:%.*]] = call i32 @llvm.llround.i32.f32(float [[X_I0]]) +; CHECK-NEXT: [[X_I1:%.*]] = extractelement <2 x float> [[X]], i64 1 +; CHECK-NEXT: [[RND_I1:%.*]] = call i32 @llvm.llround.i32.f32(float [[X_I1]]) +; CHECK-NEXT: [[RND_UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[RND_I0]], i64 0 +; CHECK-NEXT: [[RND:%.*]] = insertelement <2 x i32> [[RND_UPTO0]], i32 [[RND_I1]], i64 1 +; CHECK-NEXT: ret <2 x i32> [[RND]] +; + %rnd = call <2 x i32> @llvm.llround.v2i32.v2f32(<2 x float> %x) + ret <2 x i32> %rnd +} + define <2 x i1> @scalarize_is_fpclass(<2 x float> %x) #0 { ; CHECK-LABEL: @scalarize_is_fpclass( ; CHECK-NEXT: [[X_I0:%.*]] = extractelement <2 x float> [[X:%.*]], i64 0 diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn index d270686..5f2817c 100644 --- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn +++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn @@ -1104,14 +1104,12 @@ if (current_toolchain == default_toolchain) { "__fwd/get.h", "__fwd/ios.h", "__fwd/istream.h", - "__fwd/map.h", "__fwd/mdspan.h", "__fwd/memory.h", "__fwd/memory_resource.h", "__fwd/ostream.h", "__fwd/pair.h", "__fwd/queue.h", - "__fwd/set.h", "__fwd/span.h", "__fwd/sstream.h", "__fwd/stack.h", diff --git a/mlir/docs/DialectConversion.md b/mlir/docs/DialectConversion.md index cf577ec..556e73c2 100644 --- a/mlir/docs/DialectConversion.md +++ b/mlir/docs/DialectConversion.md @@ -202,17 +202,62 @@ struct MyConversionPattern : public ConversionPattern { #### Type Safety -The types of the remapped operands provided to a conversion pattern must be of a -type expected by the pattern. The expected types of a pattern are determined by -a provided [TypeConverter](#type-converter). If no type converter is provided, -the types of the remapped operands are expected to match the types of the -original operands. If a type converter is provided, the types of the remapped -operands are expected to be legal as determined by the converter. If the -remapped operand types are not of an expected type, and a materialization to the -expected type could not be performed, the pattern fails application before the -`matchAndRewrite` hook is invoked. This ensures that patterns do not have to -explicitly ensure type safety, or sanitize the types of the incoming remapped -operands. More information on type conversion is detailed in the +The types of the remapped operands provided to a conversion pattern (through +the adaptor or `ArrayRef` of operands) depend on type conversion rules. + +If the pattern was initialized with a [type converter](#type-converter), the +conversion driver passes values whose types match the legalized types of the +operands of the matched operation as per the type converter. To that end, the +conversion driver may insert target materializations to convert the most +recently mapped values to the expected legalized types. The driver tries to +reuse existing materializations on a best-effort basis, but this is not +guaranteed by the infrastructure. If the operand types of the matched op could +not be legalized, the pattern fails to apply before the `matchAndRewrite` hook +is invoked. + +Example: +```c++ +// Type converter that converts all FloatTypes to IntegerTypes. +TypeConverter converter; +converter.addConversion([](FloatType t) { + return IntegerType::get(t.getContext(), t.getWidth()); +}); + +// Assuming that `MyConversionPattern` was initialized with `converter`. +struct MyConversionPattern : public ConversionPattern { + virtual LogicalResult + matchAndRewrite(Operation *op, ArrayRef<Value> operands, /* ... */) const { +// ^^^^^^^^ +// If `op` has a FloatType operand, the respective value in `operands` +// is guaranteed to have the legalized IntegerType. If another pattern +// previously replaced the operand SSA value with an SSA value of the +// legalized type (via "replaceOp" or "applySignatureConversion"), you +// will get that SSA value directly (unless the replacement value was +// also replaced). Otherwise, you will get a materialization to the +// legalized type. +``` + +If the pattern was initialized without a type converter, the conversion driver +passes the most recently mapped values to the pattern, excluding any +materializations. If a value with the same type as the original operand is +desired, users can directly take the respective operand from the matched +operation. + +Example: When initializing the pattern from the above example without a type +converter, `operands` contains the most recent replacement values, regardless +of their types. + +Note: When running without a type converter, materializations are intentionally +excluded from the lookup process because their presence may depend on other +patterns. Passing materializations would make the conversion infrastructure +fragile and unpredictable. Moreover, there could be multiple materializations +to different types. (This can be the case when multiple patterns are running +with different type converters.) In such a case, it would be unclear which +materialization to pass. + +The above rules ensure that patterns do not have to explicitly ensure type +safety, or sanitize the types of the incoming remapped operands. More +information on type conversion is detailed in the [dedicated section](#type-conversion) below. ## Type Conversion diff --git a/mlir/include/mlir/Dialect/Linalg/Passes.td b/mlir/include/mlir/Dialect/Linalg/Passes.td index 373842c..f236629 100644 --- a/mlir/include/mlir/Dialect/Linalg/Passes.td +++ b/mlir/include/mlir/Dialect/Linalg/Passes.td @@ -89,6 +89,45 @@ def LinalgInlineScalarOperandsPass : Pass<"linalg-inline-scalar-operands"> { ]; } +def LinalgMorphOpsPass : Pass<"linalg-morph-ops"> { + let summary = "Convert named op to category ops or generic and vice-versa"; + + let description = [{ + Convert a linalg op from one representation to another equivalent. + For example, a linalg named op `linalg.add` can also be written as an + category op `linalg.elementwise`, and can also be re-written as + a `linalg.generic`, giving the morphism: + + named-op <--> category_op (elementwise, contraction, ..) <--> generic + + Note that the set of `linalg.generic` subsumes named and category ops + and therefore not all `linalg.genric` can be converted to named or + category op. Similarly, catgory ops subsume named ops. + + Note: + Legacy converters: + `--linalg-generalize-named-ops` is the path `named-op --> generic-op` + `--linalg-specialize-generic-ops` is the path `named-op <-- generic-op` + }]; + let dependentDialects = ["linalg::LinalgDialect"]; + + let options = [ + // named-op <--> category <--> generic + + // Lowering options + Option<"namedToCategory", "named-to-category", "bool", /*default=*/"false", + "convert named ops to category op e.g. `linalg.elementwise`">, + Option<"categoryToGeneric", "category-to-generic", "bool", /*default=*/"false", + "convert category ops e.g. `linalg.elementwise` to `linalg.generic`">, + Option<"namedToGeneric", "named-to-generic", "bool", /*default=*/"false", + "convert named ops e.g. `linalg.add` to `linalg.generic`">, + + // Lifting options + // TODOs: `generic-to-category`, `category-to-named` + Option<"genericToNamed", "generic-to-named", "bool", /*default=*/"false", + "convert linalg.generic to equivalent named ops"> ]; +} + def LinalgGeneralizeNamedOpsPass : Pass<"linalg-generalize-named-ops"> { let summary = "Convert named ops into generic ops"; let dependentDialects = ["linalg::LinalgDialect"]; diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h index d4ffe0a..1e5b5d4 100644 --- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h +++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h @@ -1831,6 +1831,10 @@ void populateLinalgNamedOpsGeneralizationPatterns(RewritePatternSet &patterns); void populateLinalgGenericOpsSpecializationPatterns( RewritePatternSet &patterns); +/// Populates `patterns` that convert linalg named ops e.g. `linalg.add` +/// to equivalent `linalg.elementwise`. +void populateLinalgNamedToElementwisePatterns(RewritePatternSet &patterns); + /// Populates `patterns` with patterns that fold operations like /// `linalg.transform` into elementwise op map. void populateLinalgFoldIntoElementwisePatterns(RewritePatternSet &patterns); diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h index f643765..4e651a0 100644 --- a/mlir/include/mlir/Transforms/DialectConversion.h +++ b/mlir/include/mlir/Transforms/DialectConversion.h @@ -728,6 +728,9 @@ class ConversionPatternRewriter final : public PatternRewriter { public: ~ConversionPatternRewriter() override; + /// Return the configuration of the current dialect conversion. + const ConversionConfig &getConfig() const; + /// Apply a signature conversion to given block. This replaces the block with /// a new block containing the updated signature. The operations of the given /// block are inlined into the newly-created block, which is returned. diff --git a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt index 70f846e..6ec2e9fd 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt +++ b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt @@ -23,9 +23,11 @@ add_mlir_dialect_library(MLIRLinalgTransforms InlineScalarOperands.cpp Interchange.cpp Loops.cpp + MorphOps.cpp TransposeMatmul.cpp ShardingInterfaceImpl.cpp NamedOpConversions.cpp + NamedToElementwise.cpp BlockPackMatmul.cpp PackAndUnpackPatterns.cpp Padding.cpp diff --git a/mlir/lib/Dialect/Linalg/Transforms/MorphOps.cpp b/mlir/lib/Dialect/Linalg/Transforms/MorphOps.cpp new file mode 100644 index 0000000..f261ccb --- /dev/null +++ b/mlir/lib/Dialect/Linalg/Transforms/MorphOps.cpp @@ -0,0 +1,62 @@ +//===- MorphOps.cpp - conversion between named,category and generic ops ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements conversions between linalg ops: +// named <--> category (elementwise, contraction, ..) <--> generic. +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Complex/IR/Complex.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h" +#include "mlir/Dialect/Linalg/Passes.h" +#include "mlir/Dialect/Linalg/Transforms/Transforms.h" +#include "mlir/Dialect/Math/IR/Math.h" +#include "mlir/IR/PatternMatch.h" +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" + +namespace mlir { +#define GEN_PASS_DEF_LINALGMORPHOPSPASS +#include "mlir/Dialect/Linalg/Passes.h.inc" +} // namespace mlir + +#define DEBUG_TYPE "linalg-morphism" + +using namespace mlir; +using namespace mlir::linalg; + +namespace { +struct LinalgMorphOpsPass + : public impl::LinalgMorphOpsPassBase<LinalgMorphOpsPass> { + + using impl::LinalgMorphOpsPassBase< + LinalgMorphOpsPass>::LinalgMorphOpsPassBase; + + void runOnOperation() override; +}; + +void LinalgMorphOpsPass::runOnOperation() { + + RewritePatternSet patterns(&getContext()); + + // Lowering paths (named -> category -> generic) + if (namedToCategory) { + populateLinalgNamedToElementwisePatterns(patterns); + } + if (namedToGeneric || categoryToGeneric) { + populateLinalgNamedOpsGeneralizationPatterns(patterns); + } + + // Lifting paths (named <- category <- generic) + if (genericToNamed) { + populateLinalgGenericOpsSpecializationPatterns(patterns); + } + + if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) + signalPassFailure(); +} +} // namespace diff --git a/mlir/lib/Dialect/Linalg/Transforms/NamedToElementwise.cpp b/mlir/lib/Dialect/Linalg/Transforms/NamedToElementwise.cpp new file mode 100644 index 0000000..00a076b --- /dev/null +++ b/mlir/lib/Dialect/Linalg/Transforms/NamedToElementwise.cpp @@ -0,0 +1,98 @@ +//===- NamedToElementwise.cpp - convert linalg named op into elementwise --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements rewriting those linalg named ops that are essentially +// elementwise e.g. `linalg.exp`, to `linalg.elementwise`. This allows further +// optimization on `linalg.elementwise` such as folding transpose, broadcast. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Linalg/Passes.h" +#include "mlir/Dialect/Linalg/Transforms/Transforms.h" +#include "mlir/IR/PatternMatch.h" +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/TypeSwitch.h" + +using namespace mlir; +using namespace mlir::linalg; + +#define DEBUG_TYPE "linalg-named-to-elementwise" + +namespace { +ElementwiseKind getKind(Operation *op) { + return llvm::TypeSwitch<Operation *, ElementwiseKind>(op) + .Case([](SelectOp) { return ElementwiseKind::select; }) + .Case([](AddOp) { return ElementwiseKind::add; }) + .Case([](SubOp) { return ElementwiseKind::sub; }) + .Case([](MulOp) { return ElementwiseKind::mul; }) + .Case([](DivOp) { return ElementwiseKind::div; }) + .Case([](DivUnsignedOp) { return ElementwiseKind::div_unsigned; }) + .Case([](PowFOp) { return ElementwiseKind::powf; }) + .Case([](ExpOp) { return ElementwiseKind::exp; }) + .Case([](LogOp) { return ElementwiseKind::log; }) + .Case([](AbsOp) { return ElementwiseKind::abs; }) + .Case([](CeilOp) { return ElementwiseKind::ceil; }) + .Case([](FloorOp) { return ElementwiseKind::floor; }) + .Case([](NegFOp) { return ElementwiseKind::negf; }) + .Case([](ReciprocalOp) { return ElementwiseKind::reciprocal; }) + .Case([](RoundOp) { return ElementwiseKind::round; }) + .Case([](SqrtOp) { return ElementwiseKind::sqrt; }) + .Case([](RsqrtOp) { return ElementwiseKind::rsqrt; }) + .Case([](SquareOp) { return ElementwiseKind::square; }) + .Case([](TanhOp) { return ElementwiseKind::tanh; }) + .Case([](ErfOp) { return ElementwiseKind::erf; }) + .Default([&](Operation *op) { + llvm_unreachable("unhandled case in named to elementwise"); + return ElementwiseKind::sub; + }); +} + +template <typename NamedOpTy> +struct NamedToElementwisePattern : public OpRewritePattern<NamedOpTy> { + using OpRewritePattern<NamedOpTy>::OpRewritePattern; + + LogicalResult matchAndRewrite(NamedOpTy op, + PatternRewriter &rewriter) const override { + SmallVector<NamedAttribute> attrs; + auto kindAttr = ElementwiseKindAttr::get(op.getContext(), getKind(op)); + attrs.push_back(rewriter.getNamedAttr("kind", kindAttr)); + attrs.push_back( + rewriter.getNamedAttr("indexing_maps", op.getIndexingMaps())); + + rewriter.replaceOpWithNewOp<ElementwiseOp>(op, op.getDpsInputs(), + op.getDpsInits(), attrs); + return success(); + } +}; +} // namespace + +void mlir::linalg::populateLinalgNamedToElementwisePatterns( + RewritePatternSet &patterns) { + patterns.add<NamedToElementwisePattern<SelectOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<AddOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<SubOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<MulOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<DivOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<DivUnsignedOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<PowFOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<ExpOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<LogOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<AbsOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<CeilOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<FloorOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<NegFOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<ReciprocalOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<RoundOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<SqrtOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<RsqrtOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<SquareOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<TanhOp>>(patterns.getContext()); + patterns.add<NamedToElementwisePattern<ErfOp>>(patterns.getContext()); +} diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp index f23c619..0c26b4e 100644 --- a/mlir/lib/Transforms/Utils/DialectConversion.cpp +++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp @@ -121,17 +121,8 @@ struct ConversionValueMapping { /// false positives. bool isMappedTo(Value value) const { return mappedTo.contains(value); } - /// Lookup the most recently mapped values with the desired types in the - /// mapping. - /// - /// Special cases: - /// - If the desired type range is empty, simply return the most recently - /// mapped values. - /// - If there is no mapping to the desired types, also return the most - /// recently mapped values. - /// - If there is no mapping for the given values at all, return the given - /// value. - ValueVector lookupOrDefault(Value from, TypeRange desiredTypes = {}) const; + /// Lookup a value in the mapping. + ValueVector lookup(const ValueVector &from) const; template <typename T> struct IsValueVector : std::is_same<std::decay_t<T>, ValueVector> {}; @@ -185,54 +176,31 @@ private: }; } // namespace -ValueVector -ConversionValueMapping::lookupOrDefault(Value from, - TypeRange desiredTypes) const { - // Try to find the deepest values that have the desired types. If there is no - // such mapping, simply return the deepest values. - ValueVector desiredValue; - ValueVector current{from}; - do { - // Store the current value if the types match. - if (TypeRange(ValueRange(current)) == desiredTypes) - desiredValue = current; - - // If possible, Replace each value with (one or multiple) mapped values. - ValueVector next; - for (Value v : current) { - auto it = mapping.find({v}); - if (it != mapping.end()) { - llvm::append_range(next, it->second); - } else { - next.push_back(v); - } - } - if (next != current) { - // If at least one value was replaced, continue the lookup from there. - current = std::move(next); - continue; - } - - // Otherwise: Check if there is a mapping for the entire vector. Such - // mappings are materializations. (N:M mapping are not supported for value - // replacements.) - // - // Note: From a correctness point of view, materializations do not have to - // be stored (and looked up) in the mapping. But for performance reasons, - // we choose to reuse existing IR (when possible) instead of creating it - // multiple times. - auto it = mapping.find(current); - if (it == mapping.end()) { - // No mapping found: The lookup stops here. - break; - } - current = it->second; - } while (true); +/// Marker attribute for pure type conversions. I.e., mappings whose only +/// purpose is to resolve a type mismatch. (In contrast, mappings that point to +/// the replacement values of a "replaceOp" call, etc., are not pure type +/// conversions.) +static const StringRef kPureTypeConversionMarker = "__pure_type_conversion__"; + +/// A vector of values is a pure type conversion if all values are defined by +/// the same operation and the operation has the `kPureTypeConversionMarker` +/// attribute. +static bool isPureTypeConversion(const ValueVector &values) { + assert(!values.empty() && "expected non-empty value vector"); + Operation *op = values.front().getDefiningOp(); + for (Value v : llvm::drop_begin(values)) + if (v.getDefiningOp() != op) + return false; + return op && op->hasAttr(kPureTypeConversionMarker); +} - // If the desired values were found use them, otherwise default to the leaf - // values. - // Note: If `desiredTypes` is empty, this function always returns `current`. - return !desiredValue.empty() ? std::move(desiredValue) : std::move(current); +ValueVector ConversionValueMapping::lookup(const ValueVector &from) const { + auto it = mapping.find(from); + if (it == mapping.end()) { + // No mapping found: The lookup stops here. + return {}; + } + return it->second; } //===----------------------------------------------------------------------===// @@ -930,7 +898,11 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener { /// recently mapped values. /// - If there is no mapping for the given values at all, return the given /// value. - ValueVector lookupOrDefault(Value from, TypeRange desiredTypes = {}) const; + /// + /// If `skipPureTypeConversions` is "true", materializations that are pure + /// type conversions are not considered. + ValueVector lookupOrDefault(Value from, TypeRange desiredTypes = {}, + bool skipPureTypeConversions = false) const; /// Lookup the given value within the map, or return an empty vector if the /// value is not mapped. If it is mapped, this follows the same behavior @@ -993,11 +965,19 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener { /// If `valuesToMap` is set to a non-null Value, then that value is mapped to /// the results of the unresolved materialization in the conversion value /// mapping. + /// + /// If `isPureTypeConversion` is "true", the materialization is created only + /// to resolve a type mismatch. That means it is not a regular value + /// replacement issued by the user. (Replacement values that are created + /// "out of thin air" appear like unresolved materializations because they are + /// unrealized_conversion_cast ops. However, they must be treated like + /// regular value replacements.) ValueRange buildUnresolvedMaterialization( MaterializationKind kind, OpBuilder::InsertPoint ip, Location loc, ValueVector valuesToMap, ValueRange inputs, TypeRange outputTypes, Type originalType, const TypeConverter *converter, - UnrealizedConversionCastOp *castOp = nullptr); + UnrealizedConversionCastOp *castOp = nullptr, + bool isPureTypeConversion = true); /// Find a replacement value for the given SSA value in the conversion value /// mapping. The replacement value must have the same type as the given SSA @@ -1264,10 +1244,77 @@ void ConversionPatternRewriterImpl::applyRewrites() { // State Management //===----------------------------------------------------------------------===// -ValueVector -ConversionPatternRewriterImpl::lookupOrDefault(Value from, - TypeRange desiredTypes) const { - return mapping.lookupOrDefault(from, desiredTypes); +ValueVector ConversionPatternRewriterImpl::lookupOrDefault( + Value from, TypeRange desiredTypes, bool skipPureTypeConversions) const { + // Helper function that looks up each value in `values` individually and then + // composes the results. If that fails, it tries to look up the entire vector + // at once. + auto composedLookup = [&](const ValueVector &values) -> ValueVector { + // If possible, replace each value with (one or multiple) mapped values. + ValueVector next; + for (Value v : values) { + ValueVector r = mapping.lookup({v}); + if (!r.empty()) { + llvm::append_range(next, r); + } else { + next.push_back(v); + } + } + if (next != values) { + // At least one value was replaced. + return next; + } + + // Otherwise: Check if there is a mapping for the entire vector. Such + // mappings are materializations. (N:M mapping are not supported for value + // replacements.) + // + // Note: From a correctness point of view, materializations do not have to + // be stored (and looked up) in the mapping. But for performance reasons, + // we choose to reuse existing IR (when possible) instead of creating it + // multiple times. + ValueVector r = mapping.lookup(values); + if (r.empty()) { + // No mapping found: The lookup stops here. + return {}; + } + return r; + }; + + // Try to find the deepest values that have the desired types. If there is no + // such mapping, simply return the deepest values. + ValueVector desiredValue; + ValueVector current{from}; + ValueVector lastNonMaterialization{from}; + do { + // Store the current value if the types match. + bool match = TypeRange(ValueRange(current)) == desiredTypes; + if (skipPureTypeConversions) { + // Skip pure type conversions, if requested. + bool pureConversion = isPureTypeConversion(current); + match &= !pureConversion; + // Keep track of the last mapped value that was not a pure type + // conversion. + if (!pureConversion) + lastNonMaterialization = current; + } + if (match) + desiredValue = current; + + // Lookup next value in the mapping. + ValueVector next = composedLookup(current); + if (next.empty()) + break; + current = std::move(next); + } while (true); + + // If the desired values were found use them, otherwise default to the leaf + // values. (Skip pure type conversions, if requested.) + if (!desiredTypes.empty()) + return desiredValue; + if (skipPureTypeConversions) + return lastNonMaterialization; + return current; } ValueVector @@ -1324,10 +1371,13 @@ LogicalResult ConversionPatternRewriterImpl::remapValues( Location operandLoc = inputLoc ? *inputLoc : operand.getLoc(); if (!currentTypeConverter) { - // The current pattern does not have a type converter. I.e., it does not - // distinguish between legal and illegal types. For each operand, simply - // pass through the most recently mapped values. - remapped.push_back(lookupOrDefault(operand)); + // The current pattern does not have a type converter. Pass the most + // recently mapped values, excluding materializations. Materializations + // are intentionally excluded because their presence may depend on other + // patterns. Including materializations would make the lookup fragile + // and unpredictable. + remapped.push_back(lookupOrDefault(operand, /*desiredTypes=*/{}, + /*skipPureTypeConversions=*/true)); continue; } @@ -1356,7 +1406,8 @@ LogicalResult ConversionPatternRewriterImpl::remapValues( } // Create a materialization for the most recently mapped values. - repl = lookupOrDefault(operand); + repl = lookupOrDefault(operand, /*desiredTypes=*/{}, + /*skipPureTypeConversions=*/true); ValueRange castValues = buildUnresolvedMaterialization( MaterializationKind::Target, computeInsertPoint(repl), operandLoc, /*valuesToMap=*/repl, /*inputs=*/repl, /*outputTypes=*/legalTypes, @@ -1482,7 +1533,8 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion( OpBuilder::InsertPoint(newBlock, newBlock->begin()), origArg.getLoc(), /*valuesToMap=*/{}, /*inputs=*/ValueRange(), - /*outputTypes=*/origArgType, /*originalType=*/Type(), converter) + /*outputTypes=*/origArgType, /*originalType=*/Type(), converter, + /*castOp=*/nullptr, /*isPureTypeConversion=*/false) .front(); replaceUsesOfBlockArgument(origArg, mat, converter); continue; @@ -1523,7 +1575,7 @@ ValueRange ConversionPatternRewriterImpl::buildUnresolvedMaterialization( MaterializationKind kind, OpBuilder::InsertPoint ip, Location loc, ValueVector valuesToMap, ValueRange inputs, TypeRange outputTypes, Type originalType, const TypeConverter *converter, - UnrealizedConversionCastOp *castOp) { + UnrealizedConversionCastOp *castOp, bool isPureTypeConversion) { assert((!originalType || kind == MaterializationKind::Target) && "original type is valid only for target materializations"); assert(TypeRange(inputs) != outputTypes && @@ -1535,6 +1587,8 @@ ValueRange ConversionPatternRewriterImpl::buildUnresolvedMaterialization( builder.setInsertionPoint(ip.getBlock(), ip.getPoint()); auto convertOp = UnrealizedConversionCastOp::create(builder, loc, outputTypes, inputs); + if (isPureTypeConversion) + convertOp->setAttr(kPureTypeConversionMarker, builder.getUnitAttr()); if (!valuesToMap.empty()) mapping.map(valuesToMap, convertOp.getResults()); if (castOp) @@ -1650,7 +1704,8 @@ void ConversionPatternRewriterImpl::replaceOp( MaterializationKind::Source, computeInsertPoint(result), result.getLoc(), /*valuesToMap=*/{result}, /*inputs=*/ValueRange(), /*outputTypes=*/result.getType(), /*originalType=*/Type(), - currentTypeConverter); + currentTypeConverter, /*castOp=*/nullptr, + /*isPureTypeConversion=*/false); continue; } @@ -1754,6 +1809,10 @@ ConversionPatternRewriter::ConversionPatternRewriter( ConversionPatternRewriter::~ConversionPatternRewriter() = default; +const ConversionConfig &ConversionPatternRewriter::getConfig() const { + return impl->config; +} + void ConversionPatternRewriter::replaceOp(Operation *op, Operation *newOp) { assert(op && newOp && "expected non-null op"); replaceOp(op, newOp->getResults()); @@ -1895,7 +1954,7 @@ void ConversionPatternRewriter::inlineBlockBefore(Block *source, Block *dest, // ops should be moved one-by-one ("slow path"), so that a separate // `MoveOperationRewrite` is enqueued for each moved op. Moving ops in bulk is // a bit more efficient, so we try to do that when possible. - bool fastPath = !impl->config.listener; + bool fastPath = !getConfig().listener; if (fastPath) impl->inlineBlockBefore(source, dest, before); @@ -2018,8 +2077,7 @@ public: using LegalizationAction = ConversionTarget::LegalizationAction; OperationLegalizer(const ConversionTarget &targetInfo, - const FrozenRewritePatternSet &patterns, - const ConversionConfig &config); + const FrozenRewritePatternSet &patterns); /// Returns true if the given operation is known to be illegal on the target. bool isIllegal(Operation *op) const; @@ -2116,16 +2174,12 @@ private: /// The pattern applicator to use for conversions. PatternApplicator applicator; - - /// Dialect conversion configuration. - const ConversionConfig &config; }; } // namespace OperationLegalizer::OperationLegalizer(const ConversionTarget &targetInfo, - const FrozenRewritePatternSet &patterns, - const ConversionConfig &config) - : target(targetInfo), applicator(patterns), config(config) { + const FrozenRewritePatternSet &patterns) + : target(targetInfo), applicator(patterns) { // The set of patterns that can be applied to illegal operations to transform // them into legal ones. DenseMap<OperationName, LegalizationPatterns> legalizerPatterns; @@ -2286,7 +2340,7 @@ OperationLegalizer::legalizeWithFold(Operation *op, LLVM_DEBUG(logFailure(rewriterImpl.logger, "failed to legalize generated constant '{0}'", newOp->getName())); - if (!config.allowPatternRollback) { + if (!rewriter.getConfig().allowPatternRollback) { // Rolling back a folder is like rolling back a pattern. llvm::report_fatal_error( "op '" + opName + @@ -2302,10 +2356,36 @@ OperationLegalizer::legalizeWithFold(Operation *op, return success(); } +/// Report a fatal error indicating that newly produced or modified IR could +/// not be legalized. +static void +reportNewIrLegalizationFatalError(const Pattern &pattern, + const SetVector<Operation *> &newOps, + const SetVector<Operation *> &modifiedOps, + const SetVector<Block *> &insertedBlocks) { + auto newOpNames = llvm::map_range( + newOps, [](Operation *op) { return op->getName().getStringRef(); }); + auto modifiedOpNames = llvm::map_range( + modifiedOps, [](Operation *op) { return op->getName().getStringRef(); }); + StringRef detachedBlockStr = "(detached block)"; + auto insertedBlockNames = llvm::map_range(insertedBlocks, [&](Block *block) { + if (block->getParentOp()) + return block->getParentOp()->getName().getStringRef(); + return detachedBlockStr; + }); + llvm::report_fatal_error( + "pattern '" + pattern.getDebugName() + + "' produced IR that could not be legalized. " + "new ops: {" + + llvm::join(newOpNames, ", ") + "}, " + "modified ops: {" + + llvm::join(modifiedOpNames, ", ") + "}, " + "inserted block into ops: {" + + llvm::join(insertedBlockNames, ", ") + "}"); +} + LogicalResult OperationLegalizer::legalizeWithPattern(Operation *op, ConversionPatternRewriter &rewriter) { auto &rewriterImpl = rewriter.getImpl(); + const ConversionConfig &config = rewriter.getConfig(); #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS Operation *checkOp; @@ -2389,8 +2469,8 @@ OperationLegalizer::legalizeWithPattern(Operation *op, appliedPatterns.erase(&pattern); if (failed(result)) { if (!rewriterImpl.config.allowPatternRollback) - llvm::report_fatal_error("pattern '" + pattern.getDebugName() + - "' produced IR that could not be legalized"); + reportNewIrLegalizationFatalError(pattern, newOps, modifiedOps, + insertedBlocks); rewriterImpl.resetState(curState, pattern.getDebugName()); } if (config.listener) @@ -2749,8 +2829,7 @@ struct OperationConverter { const FrozenRewritePatternSet &patterns, const ConversionConfig &config, OpConversionMode mode) - : config(config), opLegalizer(target, patterns, this->config), - mode(mode) {} + : config(config), opLegalizer(target, patterns), mode(mode) {} /// Converts the given operations to the conversion target. LogicalResult convertOperations(ArrayRef<Operation *> ops); @@ -2902,6 +2981,10 @@ LogicalResult OperationConverter::convertOperations(ArrayRef<Operation *> ops) { SmallVector<UnrealizedConversionCastOp> remainingCastOps; reconcileUnrealizedCasts(allCastOps, &remainingCastOps); + // Drop markers. + for (UnrealizedConversionCastOp castOp : remainingCastOps) + castOp->removeAttr(kPureTypeConversionMarker); + // Try to legalize all unresolved materializations. if (config.buildMaterializations) { IRRewriter rewriter(rewriterImpl.context, config.listener); diff --git a/mlir/test/Dialect/Linalg/elementwise/named-to-elementwise.mlir b/mlir/test/Dialect/Linalg/elementwise/named-to-elementwise.mlir new file mode 100644 index 0000000..2332b28 --- /dev/null +++ b/mlir/test/Dialect/Linalg/elementwise/named-to-elementwise.mlir @@ -0,0 +1,56 @@ +// RUN: mlir-opt %s -linalg-morph-ops=named-to-category -split-input-file | FileCheck %s + +// CHECK: @exp(%[[A:.+]]: tensor<16x8xf32>, %[[B:.+]]: tensor<16x8xf32>) -> tensor<16x8xf32> { +// CHECK: {{.*}} = linalg.elementwise +// CHECK-SAME: kind=#linalg.elementwise_kind<exp> +// CHECK-SAME: ins(%[[A]] : tensor<16x8xf32>) +// CHECK-SAME: outs(%[[B]] : tensor<16x8xf32>) -> tensor<16x8xf32> +// +func.func @exp(%A : tensor<16x8xf32>, %B : tensor<16x8xf32>) -> tensor<16x8xf32> { + %exp = linalg.exp ins(%A : tensor<16x8xf32>) outs(%B : tensor<16x8xf32>) -> tensor<16x8xf32> + return %exp : tensor<16x8xf32> +} + +// ---- + +// CHECK: @add(%[[A:.+]]: tensor<16x8xf32>, %[[B:.+]]: tensor<16x8xf32>, %[[C:.+]]: tensor<16x8xf32>) -> tensor<16x8xf32> { +// CHECK: {{.*}} = linalg.elementwise +// CHECK-SAME: kind=#linalg.elementwise_kind<add> +// CHECK-SAME: ins(%[[A]], %[[B]] : tensor<16x8xf32>, tensor<16x8xf32>) +// CHECK-SAME: outs(%[[C]] : tensor<16x8xf32>) -> tensor<16x8xf32> +// +func.func @add(%A : tensor<16x8xf32>, %B: tensor<16x8xf32>, %C : tensor<16x8xf32>) -> tensor<16x8xf32> { + %add = linalg.add ins(%A, %B : tensor<16x8xf32>, tensor<16x8xf32>) outs(%C : tensor<16x8xf32>) -> tensor<16x8xf32> + return %add : tensor<16x8xf32> +} + +// ---- + +// CHECK: @sub(%[[A:.+]]: tensor<16x8xf32>, %[[B:.+]]: tensor<16x8xf32>, %[[C:.+]]: tensor<16x8xf32>) -> tensor<16x8xf32> { +// CHECK: {{.*}} = linalg.elementwise +// CHECK-SAME: kind=#linalg.elementwise_kind<sub> +// CHECK-SAME: ins(%[[A]], %[[B]] : tensor<16x8xf32>, tensor<16x8xf32>) +// CHECK-SAME: outs(%[[C]] : tensor<16x8xf32>) +// +func.func @sub(%A : tensor<16x8xf32>, %B: tensor<16x8xf32>, %C : tensor<16x8xf32>) -> tensor<16x8xf32> { + %sub = linalg.sub ins(%A, %B : tensor<16x8xf32>, tensor<16x8xf32>) outs(%C : tensor<16x8xf32>) -> tensor<16x8xf32> + return %sub : tensor<16x8xf32> +} + +// ---- + +// CHECK: @ternary_select(%[[A:.+]]: tensor<4x8x16xi1>, %[[B:.+]]: tensor<4x8x16xf32>, %[[C:.+]]: tensor<4x8x16xf32>) +// CHECK: %[[E:.+]] = tensor.empty() : tensor<4x8x16xf32> +// CHECK: {{.*}} = linalg.elementwise +// CHECK-SAME: kind=#linalg.elementwise_kind<select> +// CHECK-SAME: ins(%[[A]], %[[B]], %[[C]] : tensor<4x8x16xi1>, tensor<4x8x16xf32>, tensor<4x8x16xf32>) +// CHECK-SAME: outs(%[[E]] : tensor<4x8x16xf32>) -> tensor<4x8x16xf32> +// +func.func @ternary_select(%A: tensor<4x8x16xi1>, %B: tensor<4x8x16xf32>, %C: tensor<4x8x16xf32>) + -> tensor<4x8x16xf32> { + %empty = tensor.empty() : tensor<4x8x16xf32> + %select = linalg.select + ins(%A, %B, %C : tensor<4x8x16xi1>, tensor<4x8x16xf32>, tensor<4x8x16xf32>) + outs(%empty: tensor<4x8x16xf32>) -> tensor<4x8x16xf32> + return %select : tensor<4x8x16xf32> +} diff --git a/mlir/test/Dialect/Linalg/linalg-morph-category-ops.mlir b/mlir/test/Dialect/Linalg/linalg-morph-category-ops.mlir new file mode 100644 index 0000000..00602c4 --- /dev/null +++ b/mlir/test/Dialect/Linalg/linalg-morph-category-ops.mlir @@ -0,0 +1,15 @@ +// Forward path `named -> category -> generic` +// RUN: mlir-opt %s -linalg-morph-ops=named-to-category | FileCheck %s --check-prefix=NAMED_TO_CATEGORY + +// RUN: mlir-opt %s -linalg-morph-ops=named-to-category | \ +// RUN: mlir-opt %s -linalg-morph-ops=category-to-generic | FileCheck %s --check-prefix=CATEGORY_TO_GENERIC + +func.func @exp(%A : tensor<16x8xf32>, %B : tensor<16x8xf32>) -> tensor<16x8xf32> { + %exp = linalg.exp ins(%A : tensor<16x8xf32>) outs(%B : tensor<16x8xf32>) -> tensor<16x8xf32> + return %exp : tensor<16x8xf32> +} +// NAMED_TO_CATEGORY: linalg.elementwise +// NAMED_TO_CATEGORY-NOT: linalg.exp + +// CATEGORY_TO_GENERIC: linalg.generic +// CATEGORY_TO_GENERIC-NOT: linalg.elementwise diff --git a/mlir/test/Dialect/Linalg/linalg-morph-multi-step.mlir b/mlir/test/Dialect/Linalg/linalg-morph-multi-step.mlir new file mode 100644 index 0000000..ab50a44 --- /dev/null +++ b/mlir/test/Dialect/Linalg/linalg-morph-multi-step.mlir @@ -0,0 +1,14 @@ +// RUN: mlir-opt %s -linalg-morph-ops=named-to-generic | FileCheck %s --check-prefix=NAMED_TO_GENERIC +// RUN: mlir-opt %s -linalg-morph-ops=named-to-generic | mlir-opt %s -linalg-morph-ops=generic-to-named | \ +// RUN: FileCheck %s --check-prefix=ROUND_TRIP + +func.func @exp(%A : tensor<16x8xf32>, %B : tensor<16x8xf32>) -> tensor<16x8xf32> { + %exp = linalg.exp ins(%A : tensor<16x8xf32>) outs(%B : tensor<16x8xf32>) -> tensor<16x8xf32> + return %exp : tensor<16x8xf32> +} + +// NAMED_TO_GENERIC: linalg.generic +// NAMED_TO_GENERIC-NOT: linalg.exp + +// ROUND_TRIP: linalg.exp +// ROUND_TRIP-NOT: linalg.generic diff --git a/mlir/test/Transforms/test-legalizer.mlir b/mlir/test/Transforms/test-legalizer.mlir index e4406e6..5630d15 100644 --- a/mlir/test/Transforms/test-legalizer.mlir +++ b/mlir/test/Transforms/test-legalizer.mlir @@ -415,3 +415,20 @@ func.func @test_multiple_1_to_n_replacement() { %0 = "test.multiple_1_to_n_replacement"() : () -> (f16) "test.invalid"(%0) : (f16) -> () } + +// ----- + +// CHECK-LABEL: func @test_lookup_without_converter +// CHECK: %[[producer:.*]] = "test.valid_producer"() : () -> i16 +// CHECK: %[[cast:.*]] = "test.cast"(%[[producer]]) : (i16) -> f64 +// CHECK: "test.valid_consumer"(%[[cast]]) : (f64) -> () +// CHECK: "test.valid_consumer"(%[[producer]]) : (i16) -> () +func.func @test_lookup_without_converter() { + %0 = "test.replace_with_valid_producer"() {type = i16} : () -> (i64) + "test.replace_with_valid_consumer"(%0) {with_converter} : (i64) -> () + // Make sure that the second "replace_with_valid_consumer" lowering does not + // lookup the materialization that was created for the above op. + "test.replace_with_valid_consumer"(%0) : (i64) -> () + // expected-remark@+1 {{op 'func.return' is not legalizable}} + return +} diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td index 2eaad55..843bd30 100644 --- a/mlir/test/lib/Dialect/Test/TestOps.td +++ b/mlir/test/lib/Dialect/Test/TestOps.td @@ -2104,6 +2104,10 @@ def TestInvalidOp : TEST_Op<"invalid", [Terminator]>, Arguments<(ins Variadic<AnyType>)>; def TestTypeProducerOp : TEST_Op<"type_producer">, Results<(outs AnyType)>; +def TestValidProducerOp : TEST_Op<"valid_producer">, + Results<(outs AnyType)>; +def TestValidConsumerOp : TEST_Op<"valid_consumer">, + Arguments<(ins AnyType)>; def TestAnotherTypeProducerOp : TEST_Op<"another_type_producer">, Results<(outs AnyType)>; def TestTypeConsumerOp : TEST_Op<"type_consumer">, diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp index eda618f..7150401 100644 --- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp +++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp @@ -1198,6 +1198,47 @@ public: } }; +/// Pattern that replaces test.replace_with_valid_producer with +/// test.valid_producer and the specified type. +class TestReplaceWithValidProducer : public ConversionPattern { +public: + TestReplaceWithValidProducer(MLIRContext *ctx) + : ConversionPattern("test.replace_with_valid_producer", 1, ctx) {} + LogicalResult + matchAndRewrite(Operation *op, ArrayRef<Value> operands, + ConversionPatternRewriter &rewriter) const final { + auto attr = op->getAttrOfType<TypeAttr>("type"); + if (!attr) + return failure(); + rewriter.replaceOpWithNewOp<TestValidProducerOp>(op, attr.getValue()); + return success(); + } +}; + +/// Pattern that replaces test.replace_with_valid_consumer with +/// test.valid_consumer. Can be used with and without a type converter. +class TestReplaceWithValidConsumer : public ConversionPattern { +public: + TestReplaceWithValidConsumer(MLIRContext *ctx, const TypeConverter &converter) + : ConversionPattern(converter, "test.replace_with_valid_consumer", 1, + ctx) {} + TestReplaceWithValidConsumer(MLIRContext *ctx) + : ConversionPattern("test.replace_with_valid_consumer", 1, ctx) {} + + LogicalResult + matchAndRewrite(Operation *op, ArrayRef<Value> operands, + ConversionPatternRewriter &rewriter) const final { + // with_converter present: pattern must have been initialized with a type + // converter. + // with_converter absent: pattern must have been initialized without a type + // converter. + if (op->hasAttr("with_converter") != static_cast<bool>(getTypeConverter())) + return failure(); + rewriter.replaceOpWithNewOp<TestValidConsumerOp>(op, operands[0]); + return success(); + } +}; + /// This pattern matches a test.convert_block_args op. It either: /// a) Duplicates all block arguments, /// b) or: drops all block arguments and replaces each with 2x the first @@ -1314,6 +1355,7 @@ struct TestTypeConverter : public TypeConverter { TestTypeConverter() { addConversion(convertType); addSourceMaterialization(materializeCast); + addTargetMaterialization(materializeCast); } static LogicalResult convertType(Type t, SmallVectorImpl<Type> &results) { @@ -1389,10 +1431,12 @@ struct TestLegalizePatternDriver TestBoundedRecursiveRewrite, TestNestedOpCreationUndoRewrite, TestReplaceEraseOp, TestCreateUnregisteredOp, TestUndoMoveOpBefore, TestUndoPropertiesModification, TestEraseOp, + TestReplaceWithValidProducer, TestReplaceWithValidConsumer, TestRepetitive1ToNConsumer>(&getContext()); patterns.add<TestDropOpSignatureConversion, TestDropAndReplaceInvalidOp, TestPassthroughInvalidOp, TestMultiple1ToNReplacement, - TestBlockArgReplace>(&getContext(), converter); + TestBlockArgReplace, TestReplaceWithValidConsumer>( + &getContext(), converter); patterns.add<TestConvertBlockArgs>(converter, &getContext()); mlir::populateAnyFunctionOpInterfaceTypeConversionPattern(patterns, converter); @@ -1402,7 +1446,8 @@ struct TestLegalizePatternDriver ConversionTarget target(getContext()); target.addLegalOp<ModuleOp>(); target.addLegalOp<LegalOpA, LegalOpB, LegalOpC, TestCastOp, TestValidOp, - TerminatorOp, OneRegionOp>(); + TerminatorOp, OneRegionOp, TestValidProducerOp, + TestValidConsumerOp>(); target.addLegalOp(OperationName("test.legal_op", &getContext())); target .addIllegalOp<ILLegalOpF, TestRegionBuilderOp, TestOpWithRegionFold>(); diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp index 6486b2b..272a12a 100644 --- a/offload/liboffload/src/OffloadImpl.cpp +++ b/offload/liboffload/src/OffloadImpl.cpp @@ -72,6 +72,8 @@ struct ol_queue_impl_t { struct ol_event_impl_t { ol_event_impl_t(void *EventInfo, ol_queue_handle_t Queue) : EventInfo(EventInfo), Queue(Queue) {} + // EventInfo may be null, in which case the event should be considered always + // complete void *EventInfo; ol_queue_handle_t Queue; }; @@ -509,8 +511,8 @@ Error olWaitEvents_impl(ol_queue_handle_t Queue, ol_event_handle_t *Events, return Plugin::error(ErrorCode::INVALID_NULL_HANDLE, "olWaitEvents asked to wait on a NULL event"); - // Do nothing if the event is for this queue - if (Event->Queue == Queue) + // Do nothing if the event is for this queue or the event is always complete + if (Event->Queue == Queue || !Event->EventInfo) continue; if (auto Err = Device->waitEvent(Event->EventInfo, Queue->AsyncInfo)) @@ -548,6 +550,10 @@ Error olGetQueueInfoSize_impl(ol_queue_handle_t Queue, ol_queue_info_t PropName, } Error olSyncEvent_impl(ol_event_handle_t Event) { + if (!Event->EventInfo) + // Event always complete + return Plugin::success(); + if (auto Res = Event->Queue->Device->Device->syncEvent(Event->EventInfo)) return Res; @@ -555,8 +561,9 @@ Error olSyncEvent_impl(ol_event_handle_t Event) { } Error olDestroyEvent_impl(ol_event_handle_t Event) { - if (auto Res = Event->Queue->Device->Device->destroyEvent(Event->EventInfo)) - return Res; + if (Event->EventInfo) + if (auto Res = Event->Queue->Device->Device->destroyEvent(Event->EventInfo)) + return Res; return olDestroy(Event); } @@ -590,7 +597,16 @@ Error olGetEventInfoSize_impl(ol_event_handle_t Event, ol_event_info_t PropName, } Error olCreateEvent_impl(ol_queue_handle_t Queue, ol_event_handle_t *EventOut) { + auto Pending = Queue->Device->Device->hasPendingWork(Queue->AsyncInfo); + if (auto Err = Pending.takeError()) + return Err; + *EventOut = new ol_event_impl_t(nullptr, Queue); + if (!*Pending) + // Queue is empty, don't record an event and consider the event always + // complete + return Plugin::success(); + if (auto Res = Queue->Device->Device->createEvent(&(*EventOut)->EventInfo)) return Res; diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp index 82deeb4..852c0e9 100644 --- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp +++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp @@ -2591,6 +2591,17 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy { return Event->wait(*Stream); } + Expected<bool> hasPendingWorkImpl(AsyncInfoWrapperTy &AsyncInfo) override { + auto Stream = AsyncInfo.getQueueAs<AMDGPUStreamTy *>(); + if (!Stream) + return false; + + auto Query = Stream->query(); + if (Query) + return !*Query; + return Query.takeError(); + } + /// Synchronize the current thread with the event. Error syncEventImpl(void *EventPtr) override { AMDGPUEventTy *Event = reinterpret_cast<AMDGPUEventTy *>(EventPtr); diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h index 4b7d410..1d64193 100644 --- a/offload/plugins-nextgen/common/include/PluginInterface.h +++ b/offload/plugins-nextgen/common/include/PluginInterface.h @@ -973,6 +973,14 @@ struct GenericDeviceTy : public DeviceAllocatorTy { Error printInfo(); virtual Expected<InfoTreeNode> obtainInfoImpl() = 0; + /// Return true if the device has work that is either queued or currently + /// running + /// + /// Devices which cannot report this information should always return true + Expected<bool> hasPendingWork(__tgt_async_info *AsyncInfo); + virtual Expected<bool> + hasPendingWorkImpl(AsyncInfoWrapperTy &AsyncInfoWrapper) = 0; + /// Getters of the grid values. uint32_t getWarpSize() const { return GridValues.GV_Warp_Size; } uint32_t getThreadLimit() const { return GridValues.GV_Max_WG_Size; } diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp index 0a57046..bcc9179 100644 --- a/offload/plugins-nextgen/common/src/PluginInterface.cpp +++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp @@ -1626,6 +1626,21 @@ Error GenericDeviceTy::waitEvent(void *EventPtr, __tgt_async_info *AsyncInfo) { return Err; } +Expected<bool> GenericDeviceTy::hasPendingWork(__tgt_async_info *AsyncInfo) { + AsyncInfoWrapperTy AsyncInfoWrapper(*this, AsyncInfo); + auto Res = hasPendingWorkImpl(AsyncInfoWrapper); + if (auto Err = Res.takeError()) { + AsyncInfoWrapper.finalize(Err); + return Err; + } + + auto Err = Plugin::success(); + AsyncInfoWrapper.finalize(Err); + if (Err) + return Err; + return Res; +} + Error GenericDeviceTy::syncEvent(void *EventPtr) { return syncEventImpl(EventPtr); } diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp index c5f3167..7649fd9 100644 --- a/offload/plugins-nextgen/cuda/src/rtl.cpp +++ b/offload/plugins-nextgen/cuda/src/rtl.cpp @@ -916,6 +916,11 @@ struct CUDADeviceTy : public GenericDeviceTy { return Plugin::check(Res, "error in cuStreamWaitEvent: %s"); } + // TODO: This should be implementable on CUDA + Expected<bool> hasPendingWorkImpl(AsyncInfoWrapperTy &AsyncInfo) override { + return true; + } + /// Synchronize the current thread with the event. Error syncEventImpl(void *EventPtr) override { CUevent Event = reinterpret_cast<CUevent>(EventPtr); diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp index d950572..9abc350 100644 --- a/offload/plugins-nextgen/host/src/rtl.cpp +++ b/offload/plugins-nextgen/host/src/rtl.cpp @@ -333,6 +333,9 @@ struct GenELF64DeviceTy : public GenericDeviceTy { AsyncInfoWrapperTy &AsyncInfoWrapper) override { return Plugin::success(); } + Expected<bool> hasPendingWorkImpl(AsyncInfoWrapperTy &AsyncInfo) override { + return true; + } Error syncEventImpl(void *EventPtr) override { return Plugin::success(); } /// Print information about the device. |